#!/bin/ksh #============================================================================ # File: check_10gr2rac.sh # Type: Korn-shell script # Date: 16-Nov 2006 # Author: Tim Gorman (Evergreen Database Technologies, Inc) # # Description: # # Shell script to be used for initial diagnosis of the state of # "health" in an Oracle10gR2 (v10.2.x) RAC database. The script # first verifies the health of the "Cluster Ready Services" and then # verifies the health of the local RAC instance. # # Modifications: # TGorman 16-Nov 2006: adapted from earlier scripts # #============================================================================ # #---------------------------------------------------------------------------- # Set important shell variable values... #---------------------------------------------------------------------------- _Prgm=check_10gr2rac _HostName=`hostname -s` # #---------------------------------------------------------------------------- # Retrieve shell script options from the script's command-line... # # Default the script to "terse mode" and default the search criteria for # trace/log files to 180 minutes (i.e. 3 hours)... #---------------------------------------------------------------------------- _VerboseMode="FALSE" integer _FileAgeInMins=180 # while getopts hvm: OPTION do case ${OPTION} in v) _VerboseMode="TRUE" ;; m) integer _FileAgeInMins=${OPTARG} ;; *) echo "Usage: \"${_Prgm}.sh [ -h ] [ -m mins ] [ -v ]\"; aborting..." echo "\t-h\tdisplay this help text message" echo "\t-m mins\tspecify the age of log/trace files to examine in minutes (default: 180)" echo "\t-v\trun in \"verbose\" mode (default: \"terse\" mode)\n" exit 1 ;; esac done # #---------------------------------------------------------------------------- # define a shell function to print messages only when script is in "verbose # mode"... #---------------------------------------------------------------------------- _Echo() # ...definition of shell function "_Echo"... { if [[ "${_VerboseMode}" = "TRUE" ]] then echo $* fi } # ...end of definition of shell function "_Echo"... # #---------------------------------------------------------------------------- # define a shell function to "tail" files only when script is in "verbose # mode"... #---------------------------------------------------------------------------- _Tail() # ...definition of shell function "_Tail"... { if [[ "${_VerboseMode}" = "TRUE" ]] then tail $1 $2 | sed 's/^/\t/' fi } # ...end of definition of shell function "_Tail"... # #---------------------------------------------------------------------------- # define a shell function to "cat" files only when script is in "verbose # mode"... #---------------------------------------------------------------------------- _Cat() # ...definition of shell function "_Cat"... { if [[ "${_VerboseMode}" = "TRUE" ]] then cat $* | sed 's/^/\t/' fi } # ...end of definition of shell function "_Cat"... # #---------------------------------------------------------------------------- # This function returns the Astro-Julian Date. The Julian # date (JD) is a continuous count of days from 1 January 4713 BC. # The following algorithm is good from years 1801 to 2099. # See URL: # http://aa.usno.navy.mil/faq/docs/JD_Formula.html # for more information # arguments: $1 = day, $2 = month, $3 = year in format YYYY # # Obtained from "http://www.samag.com/documents/s=8284/sam0307b/0307b_l3.htm"... #---------------------------------------------------------------------------- _Get_Astro_JD() # ...definition of shell function "_Get_Astro_JD"... { typeset -i JDD JDD=$(($1-32075+1461*($3+4800+($2-14)/12)/4+367*($2-2-($2-14)/12*12)/12-3*(($3+4900+($2-14)/12)/100)/4)) echo $JDD } # ...end of definition of shell function "_Get_Astro_JD"... # #---------------------------------------------------------------------------- # This function computes the gregorian date from the julian date - $1. # Returns a date string in the form: YYYY-MM-DD # See URL: # http://aa.usno.navy.mil/faq/docs/JD_Formula.html # for more information # # Obtained from "http://www.samag.com/documents/s=8284/sam0307b/0307b_l4.htm"... #---------------------------------------------------------------------------- _Get_Greg_From_JD() # ...definition of shell function "_Get_Greg_From_JD"... { typeset -i L typeset -i N typeset -i I typeset -i J typeset -i DAY typeset -i MON typeset -i YR L=$(($1+68569)) # $1 is the julian date N=$((4*L/146097)) L=$((L-(146097*N+3)/4)) I=$((4000*(L+1)/1461001)) L=$((L-1461*I/4+31)) J=$((80*L/2447)) DAY=$((L-2447*J/80)) L=$((J/11)) MON=$((J+2-12*L)) YR=$((100*(N-49)+I+L)) echo "${YR}-${MON}-${DAY}" } # ...end of definition of shell function "_Get_Greg_From_JD"... # #---------------------------------------------------------------------------- # Location of the "ocr.loc" file is OS-dependent... #---------------------------------------------------------------------------- case `uname -s` in Linux) _OcrLocDir=/etc/oracle ;; SunOS) _OcrLocDir=/var/opt/oracle ;; *) echo "\nOS \"`uname -s`\" not supported by this script (yet); aborting...\n" exit 1 ;; esac _OcrLocFile=${_OcrLocDir}/ocr.loc # #---------------------------------------------------------------------------- # Announce the commencement of the script, with timestamp... #---------------------------------------------------------------------------- echo "\nDiagnostic script \"${_Prgm}.sh\" starting: `date`" # #---------------------------------------------------------------------------- # Fail out of the script if the following environment variables are not set... #---------------------------------------------------------------------------- _Echo "\nCheck if environment variable \"ORACLE_HOME\" is set..." if [[ "~${ORACLE_HOME}~" = "~~" ]] then echo "\n\tEnvironment variable \"ORACLE_HOME\" not set; aborting...\n" exit 1 fi # _Echo "\nCheck if environment variable \"ORACLE_SID\" is set..." if [[ "~${ORACLE_SID}~" = "~~" ]] then echo "\n\tEnvironment variable \"ORACLE_SID\" not set; aborting...\n" exit 1 fi # _Echo "\nCheck if environment variable \"DB_NAME\" is set..." if [[ "~${DB_NAME}~" = "~~" ]] then typeset -u DB_NAME="`echo $ORACLE_SID | sed 's/[0-9]*$//'`" echo -n "\n\tEnvironment variable \"DB_NAME\" not set; use value \"${DB_NAME}\" (derived from \"ORACLE_SID\")? [y/n]: " read _Response if [[ "${_Response}" != "Y" && "${_Response}" != "y" ]] then echo "\n\tEnvironment variable \"DB_NAME\" not set; aborting...\n" exit 1 fi export DB_NAME fi # #---------------------------------------------------------------------------- # At Corporate Express, there seems to be some confusion about whether # the environment variable CRS_HOME or ORA_CRS_HOME should be used to # indicate the location of the ORACLE_HOME for the CRS software stack... # # So, the following code checks as follows: # # if CRS_HOME is not set then # fail if ORA_CRS_HOME is also not set # set CRS_HOME to ORA_CRS_HOME # fi # if ORA_CRS_HOME is not set then # set ORA_CRS_HOME to CRS_HOME # fi # # By the time we do the second check (on whether ORA_CRS_HOME is set), we've # already made sure that CRS_HOME is set (or the script has failed)... # #---------------------------------------------------------------------------- _Echo "\nCheck if environment variable \"CRS_HOME\" is set..." if [[ "~${CRS_HOME}~" = "~~" ]] then if [[ "~${ORA_CRS_HOME}~" = "~~" ]] then echo "\n\tEnvironment variables \"ORA_CRS_HOME\" and \"CRS_HOME\" not set; aborting...\n" exit 1 fi _Echo "\tSetting environment variable \"CRS_HOME\" to the value of \"ORA_CRS_HOME\"; FYI..." export CRS_HOME=${ORA_CRS_HOME} fi # _Echo "\nCheck if environment variable \"ORA_CRS_HOME\" is set..." if [[ "~${ORA_CRS_HOME}~" = "~~" ]] then _Echo "\tSetting environment variable \"ORA_CRS_HOME\" to the value of \"CRS_HOME\"; FYI..." export ORA_CRS_HOME=${CRS_HOME} fi # #---------------------------------------------------------------------------- # Initialize the script's exit status to 0 (success). This value will be # incremented to a positive non-zero value for each deficiency found... #---------------------------------------------------------------------------- integer _ExitStatus=0 # #---------------------------------------------------------------------------- # Validate the presence and accessibility of important directories... #---------------------------------------------------------------------------- _Echo "\nCheck accessibility of directory \"${_OcrLocDir}\"..." if [ ! -d ${_OcrLocDir} ] then echo "\n\tDirectory \"${_OcrLocDir}\" not accessible; aborting...\n" exit 1 fi # #---------------------------------------------------------------------------- # Validate contents of "${_OcrLocFile}"... #---------------------------------------------------------------------------- _Echo "\nCheck accessibility of primary OCR file in \"${_OcrLocFile}\"..." _OcrConfigLoc="`grep ocrconfig_loc ${_OcrLocFile} | awk -F= '{print $2}'`" if [ ! -r ${_OcrConfigLoc} ] then echo "\n\tPrimary OCR file \"${_OcrConfigLoc}\" not accessible..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tPrimary OCR file \"${_OcrConfigLoc}\" is accessible - OK..." fi # _Echo "\nCheck accessibility of mirror OCR file in \"${_OcrLocFile}\"..." _OcrMirrorLoc="`grep ocrmirrorconfig_loc ${_OcrLocFile} | awk -F= '{print $2}'`" if [[ "~${_OcrMirrorLoc}~" = "~~" ]] then _Echo "\tNo mirror OCR file defined - OK..." else if [ ! -r ${_OcrMirrorLoc} ] then echo "\n\tMirror OCR file \"${_OcrMirrorLoc}\" not accessible..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tMirror OCR file \"${_OcrMirrorLoc}\" is accessible - OK..." fi fi # #---------------------------------------------------------------------------- # Files named "crsctl.*" are left in the "/tmp" directory whenever startup # problems are encountered by the CRS stack. The script will list these # files out to remind users that diagnostic info can be found here... #---------------------------------------------------------------------------- _Echo "Checking if \"crsctl.*\" files exist in \"/tmp\" directory..." _FileList="`find /tmp -name "crsctl.[0-9]*" -mmin -${_FileAgeInMins} -type f -print 2> /dev/null`" if [[ "~${_FileList}~" != "~~" ]] then for _f in `ls -1t ${_FileList}` do echo "\tFound file \"${_f}\"..." ls -l ${_f} | sed 's/^/\t\t/' echo "\tContents:" cat ${_f} | sed 's/^/\t\t/' done fi # #---------------------------------------------------------------------------- # With initial checks for the "ocr.loc" file and the existence of the OCR # files themselves established, now let's check to see if the ORA_CRS_HOME # and ORACLE_HOME exist... #---------------------------------------------------------------------------- _Echo "\nCheck accessibility of directory \"ORA_CRS_HOME\"..." if [ ! -d ${ORA_CRS_HOME} ] then echo "\n\tDirectory \"ORA_CRS_HOME\" (${ORA_CRS_HOME}) not accessible; aborting...\n" exit 1 fi _Echo "\nCheck accessibility of directory \"ORACLE_HOME\"..." if [ ! -d ${ORACLE_HOME} ] then echo "\n\tDirectory \"ORACLE_HOME\" (${ORACLE_HOME}) not accessible; aborting...\n" exit 1 fi for _SubDir in ${ORA_CRS_HOME}/bin \ ${ORACLE_HOME}/bin \ ${ORA_CRS_HOME}/log \ ${ORA_CRS_HOME}/log/${_HostName} do _Echo "\nCheck accessibility of sub-directory \"${_SubDir}\"..." if [ ! -d ${_SubDir} ] then echo "\n\tSub-directory \"${_SubDir}\" not accessible; aborting...\n" exit 1 fi done # #---------------------------------------------------------------------------- # Validate the presence and accessibility of important executables used by # this script in the ORA_CRS_HOME and ORACLE_HOME... #---------------------------------------------------------------------------- for _Exe in ${ORACLE_HOME}/bin/srvctl \ ${ORACLE_HOME}/bin/tnsping \ ${ORACLE_HOME}/bin/lsnrctl \ ${ORACLE_HOME}/bin/sqlplus \ ${ORA_CRS_HOME}/bin/crsctl \ ${ORA_CRS_HOME}/bin/onsctl \ ${ORA_CRS_HOME}/bin/crs_stat \ ${ORA_CRS_HOME}/bin/olsnodes \ ${ORA_CRS_HOME}/bin/ocrcheck \ ${ORA_CRS_HOME}/bin/ocrdump \ ${ORA_CRS_HOME}/bin/evmshow \ ${ORA_CRS_HOME}/bin/cluvfy do _Echo "\nCheck accessibility of executable \"${_Exe}\"..." if [ ! -x ${_Exe} ] then echo "\n\tExecutable \"${_Exe}\" not accessible; aborting...\n" exit 1 fi done # #---------------------------------------------------------------------------- # Begin a long series of checks on the CRS "alert.log" file, first by # verifying that it exists... #---------------------------------------------------------------------------- _CrsAlertLog=${ORA_CRS_HOME}/log/${_HostName}/alert${_HostName}.log if [ ! -r ${_CrsAlertLog} ] then echo "\n\tCRS \"alert.log\" file \"${_CrsAlertLog}\" not found; aborting...\n" exit 1 fi # #---------------------------------------------------------------------------- # Create a name for tempfiles to be used when making various checks on the # CRS "alert.log" file... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_awk_$$.out # set up temp file name # #---------------------------------------------------------------------------- # Grab the current date/time in various forms... #---------------------------------------------------------------------------- integer _YearTStamp="`date '+%Y'`" # get current date 4-digit year integer _MonTStamp="`date '+%m'`" # get current date month integer _DayTStamp="`date '+%d'`" # get current date day integer _HourTStamp="`date '+%H'`" # get current date hour integer _MinsTStamp="`date '+%M'`" # get current date mins # #---------------------------------------------------------------------------- # Obtain the "astro Julian" date for today... #---------------------------------------------------------------------------- integer _TodayJD=`_Get_Astro_JD ${_DayTStamp} ${_MonTStamp} ${_YearTStamp}` # #---------------------------------------------------------------------------- # Obtain the date in YYYY-MM-DD minus "_FileAgeInMins" minutes. Chances are # good that it's still "today", but if we're looking back 12 hours and it's # not yet noon-time, then this would be "yesterday", for example... #---------------------------------------------------------------------------- bc << __EOF__ > ${_TmpFile} 2> /dev/null (((${_TodayJD}*1440)+(${_HourTStamp}*60)+${_MinsTStamp})-${_FileAgeInMins})/1440 __EOF__ _SearchJD=`cat ${_TmpFile}` # find search start date, derived from current time _SearchDt=`_Get_Greg_From_JD ${_SearchJD}` # #---------------------------------------------------------------------------- # Obtain the hour-of-day to begin searching from... #---------------------------------------------------------------------------- bc << __EOF__ > ${_TmpFile} 2> /dev/null ((((${_TodayJD}-${_SearchJD})*1440)+(${_HourTStamp}*60)+${_MinsTStamp})-${_FileAgeInMins})/60 __EOF__ _SearchHr=`cat ${_TmpFile}` # find search start hour, derived from current time # #---------------------------------------------------------------------------- # Obtain the minute-of-hour to begin searching from... #---------------------------------------------------------------------------- bc << __EOF__ > ${_TmpFile} 2> /dev/null ((((${_TodayJD}-${_SearchJD})*1440)+(${_HourTStamp}*60)+${_MinsTStamp})-${_FileAgeInMins})%60 __EOF__ _SearchMi=`cat ${_TmpFile}` # find search start minutes, derived from current time _SearchTm=`echo ${_SearchHr} ${_SearchMi} | awk '{printf("%02d:%02d", $1, $2)}'` # #---------------------------------------------------------------------------- # Use the "awk" utility to scan through the CRS "alert.log" file and begin # printing out lines that are newer than "_FileAgeInMins" minutes ago... #---------------------------------------------------------------------------- awk '\ BEGIN {flag=0} \ { \ if (flag==0) { \ if ($1~"^20[0-9][0-9]-") { \ if ($1>="'${_SearchDt}'") { \ if ($2>="'${_SearchTm}'") { \ flag=1 \ } \ } \ } \ } \ if(flag==1) { \ print $0 \ } \ }' ${_CrsAlertLog} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\n\t\"awk\" on \"${_CrsAlertLog}\" file failed; aborting...\n" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 fi # #---------------------------------------------------------------------------- # Search the snippet of the CRS "alert.log" for the words "insufficient" and # "inaccessible", and display them if so... #---------------------------------------------------------------------------- if [ -r ${_TmpFile} ] then _TmpFile2=/tmp/${_Prgm}_awk2_$$.out # set up another temp file name for _Pattern in insufficient inaccessible do grep -i ${_Pattern} ${_TmpFile} >> ${_TmpFile2} done if [ ! -z "`cat ${_TmpFile2}`" ] then echo "\n\tCRS \"alert.log\" (\"${_CrsAlertLog}\") shows errors:" sed 's/^/\n\t\t/' ${_TmpFile2} integer _ExitStatus=${_ExitStatus}+1 echo "" fi fi rm -f ${_TmpFile} ${_TmpFile2} # #---------------------------------------------------------------------------- # Use the "crsctl" utility to query the following information about the CRS # software stack and configuration, if the script is being executed in "verbose mode"... #---------------------------------------------------------------------------- if [[ "${_VerboseMode}" = "TRUE" ]] then # for _CrsComponent in softwareversion activeversion do _Cmd="crsctl query crs ${_CrsComponent}" echo "\nRunning \"${_Cmd}\"..." $CRS_HOME/bin/${_Cmd} | sed 's/^/\t/' if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed" integer _ExitStatus=${_ExitStatus}+1 else echo "\t\t\"${_Cmd}\" executed successfully - OK" fi done # fi # #---------------------------------------------------------------------------- # Check for readability of vote disk(s)... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_crsctl_votedisk_$$.out _Cmd="crsctl query css votedisk" echo "\nRunning \"${_Cmd}\"..." $CRS_HOME/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else # _Echo "\t\"${_Cmd}\" executed successfully - OK" # for _f in `grep '^[ \t]*[0-9]*\.' ${_TmpFile} | awk '{print $3}'` do # #------------------------------------------------------------ # Verify readability of vote disk device/file... #------------------------------------------------------------ if [ ! -r ${_f} ] then echo "\tVote disk \"${_f}\" not readable" integer _ExitStatus=${_ExitStatus}+1 fi done # fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Verify the presence of the following processes belonging to the 10gR2 CRS # software stack... #---------------------------------------------------------------------------- for _PName in ocssd \ ocssd.bin \ cssd \ crsd.bin \ evmd \ evmd.bin \ evmlogger.bin \ oclsomon \ oclsomon.bin \ ons do # _Echo "\nChecking existance of the \"${_PName}\" process..." _Proc=`ps -eaf | grep crs | grep ${_PName} | grep -v grep` if [[ "~${_Proc}~" = "~~" ]] then echo "\t\"${_PName}\" process not running" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\t\"${_PName}\" process is running" fi # done # #---------------------------------------------------------------------------- # Use the "crsctl" utility to check the following components of the CRS # software stack... # # If a failure is detected, then advise the user to obtain more information # using the "crsctl debug statedump " command... #---------------------------------------------------------------------------- for _CrsComponent in crsd cssd evmd do _TmpFile=/tmp/${_Prgm}_crsctl_check_${_CrsComponent}_$$.out _Cmd="crsctl check ${_CrsComponent}" echo "\nRunning \"${_Cmd}\"..." $CRS_HOME/bin/${_Cmd} > ${_TmpFile} if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed" integer _ExitStatus=${_ExitStatus}+1 _Name="`echo ${_CrsComponent} | sed 's/d$//'`" _Echo "\t...consider using \"crsctl debug statedump ${_Name}\" for more info..." else if [ ! grep healthy ${_TmpFile} > /dev/null 2>&1 ] then echo "\t\"${_Cmd}\" failed" sed 's/^/\t/' ${_TmpFile} _Name="`echo ${_CrsComponent} | sed 's/d$//'`" _Echo "\t...consider using \"crsctl debug statedump ${_Name}\" for more info..." else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} done # #---------------------------------------------------------------------------- # Check ONS daemon using "onsctl ping"... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_onsctl_ping_${_CrsComponent}_$$.out _Cmd="onsctl ping" echo "\nRunning \"${_Cmd}\"..." $CRS_HOME/bin/${_Cmd} > ${_TmpFile} if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed" integer _ExitStatus=${_ExitStatus}+1 _Echo "\t...consider using \"onsctl debug\" for more info..." else if [ ! grep 'ons is running' ${_TmpFile} > /dev/null 2>&1 ] then echo "\t\"${_Cmd}\" failed" sed 's/^/\t/' ${_TmpFile} _Echo "\t...consider using \"onsctl debug\" for more info..." else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Copied/pasted from MetaLink note #259301.1, then edited slightly... # # Sample 10g CRS resource status query script # # Description: # - Returns formatted version of crs_stat -u in tabular format, with the # complete rsc names and filtering keywords # - The argument, $RSC_KEY, is optional and if passed to the script, will # limit the output to HA resources whose names match $RSC_KEY. # Requirements: # - $CRS_HOME should be set in your environment #---------------------------------------------------------------------------- _TmpFile1=/tmp/${_Prgm}_crs_stat_$$_1.out _TmpFile2=/tmp/${_Prgm}_crs_stat_$$_2.out ###RSC_KEY=$1 QSTAT=-u AWK=/usr/bin/awk # if not available use /usr/bin/awk # Table header:echo "" echo "\nRunning \"crs_stat $QSTAT\"..." $AWK \ 'BEGIN {printf "\t%-55s %-10s %-18s\n", "HA Resource", "Target", "State"; printf "\t%-55s %-10s %-18s\n", "-----------", "------", "-----";}' > ${_TmpFile1} # Table body: $CRS_HOME/bin/crs_stat $QSTAT | $AWK \ 'BEGIN { FS="="; state = 0; } $1~/NAME/ && $2~/'$RSC_KEY'/ {appname = $2; state=1}; state == 0 {next;} $1~/TARGET/ && state == 1 {apptarget = $2; state=2;} $1~/STATE/ && state == 2 {appstate = $2; state=3;} state == 3 {printf "%-55s %-10s %-18s\n", appname, apptarget, appstate; state=0;}' | \ sed 's/^/\t/' > ${_TmpFile2} if [[ "${_VerboseMode}" = "TRUE" ]] then cat ${_TmpFile1} ${_TmpFile2} else if [[ "~`grep -v ONLINE ${_TmpFile2}`~" != "~~" ]] then cat ${_TmpFile1} ${_TmpFile2} fi fi rm -f ${_TmpFile1} ${_TmpFile2} # #---------------------------------------------------------------------------- # Run "ocrcheck" and display the output if anything appears "wrong"... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_ocrcheck_$$.out echo "\nRunning \"ocrcheck\"..." _OcrCheckOutput="FALSE" ${ORA_CRS_HOME}/bin/ocrcheck | sed 's/^/\t/' > ${_TmpFile} 2>&1 if [[ "${_VerboseMode}" = "TRUE" ]] then cat ${_TmpFile} _OcrCheckOutput="TRUE" fi # #---------------------------------------------------------------------------- # Verify that "ocrcheck" output finishes with the line indicating successful # "integrity check"... #---------------------------------------------------------------------------- _Echo "\nValidating that \"cluster registry integrity check\" succeeded..." if [[ "~`grep 'Cluster registry integrity check succeeded' ${_TmpFile}`~" = "~~" ]] then if [[ "${_OcrCheckOutput}" = "FALSE" ]] then cat ${_TmpFile} _OcrCheckOutput="TRUE" fi echo "\n\t...please note that the OCR \"cluster registry integrity check\" failed..." integer _ExitStatus=${_ExitStatus}+1 fi # #---------------------------------------------------------------------------- # Verify that the contents of the "${_OcrLocFile}" file matches the contents # of the OCR as displayed by "ocrcheck"... #---------------------------------------------------------------------------- _Echo "\nValidating that OCR contents agree with \"${_OcrLocFile}\"..." for _OcrFile in `grep 'Device/File Name' ${_TmpFile} | awk -F: '{print $2}'` do if [[ "${_OcrConfigLoc}" != "${_OcrFile}" ]] then if [[ "${_OcrMirrorLoc}" != "${_OcrFile}" ]] then if [[ "${_OcrCheckOutput}" = "FALSE" ]] then cat ${_TmpFile} _OcrCheckOutput="TRUE" fi echo "\n\tDisplayed OCR location \"${_OcrFile}\" does not match entries in \"${_OcrLocFile}\"" echo "\tCompare to the contents of the \"${_OcrLocFile}\" file:" cat ${_OcrLocFile} | sed 's/^/\t\t/' integer _ExitStatus=${_ExitStatus}+1 fi fi done # #---------------------------------------------------------------------------- # Verify that all of the OCR files passed the "Device/File integrity check" # from "ocrcheck" successfully... #---------------------------------------------------------------------------- _Echo "\nValidating that OCR files passed \"device/file integrity check\"..." integer _CntOcrFiles=`grep 'Device/File Name' ${_TmpFile} | wc -l` integer _CntOcrFileChecks=`grep 'Device/File integrity check succeeded' ${_TmpFile} | wc -l` integer _CntOcrChecksFailed=${_CntOcrFiles}-${_CntOcrFileChecks} if (( ${_CntOcrChecksFailed} != 0 )) then if [[ "${_OcrCheckOutput}" = "FALSE" ]] then cat ${_TmpFile} _OcrCheckOutput="TRUE" fi echo "\n\t...please note that ${_CntOcrChecksFailed} OCR \"device/file integrity check\" failed..." integer _ExitStatus=${_ExitStatus}+1 fi # #---------------------------------------------------------------------------- # Check to see if the OCR is 50% or more full... #---------------------------------------------------------------------------- _Echo "\nValidating that OCR is less than 50% full..." integer _OcrUsedSpace=`grep 'Used space' ${_TmpFile} | awk '{print $5}'` integer _OcrAvailSpace=`grep 'Available space' ${_TmpFile} | awk '{print $5}'` if (( ${_OcrUsedSpace} >= ${_OcrAvailSpace} )) then if [[ "${_OcrCheckOutput}" = "FALSE" ]] then cat ${_TmpFile} _OcrCheckOutput="TRUE" fi echo "\n\t...please note that the OCR is 50% (or more) full..." integer _ExitStatus=${_ExitStatus}+1 fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Use the "cluvfy comp" utility to check status of various components of the # CRS... #---------------------------------------------------------------------------- for _Comp in clu clumgr ocr crs nodeapp do _TmpFile=/tmp/${_Prgm}_cluvfy_comp_${_Comp}_$$.out _Cmd="cluvfy comp ${_Comp} -n ${_HostName} -verbose" echo "\nRunning \"${_Cmd}\"..." ${ORA_CRS_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else if [[ "~`grep '^Verification of ' ${_TmpFile} | grep ' was successful. $'`~" = "~~" ]] then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} done # #---------------------------------------------------------------------------- # Use the "cluvfy comp sys" utility to check status of various components of # the CRS... #---------------------------------------------------------------------------- for _Comp in crs database do _TmpFile=/tmp/${_Prgm}_cluvfy_comp_sys_${_Comp}_$$.out _Cmd="cluvfy comp sys -n ${_HostName} -p ${_Comp} -r 10gR2 -osdba dba -orainv oinstall -verbose" echo "\nRunning \"${_Cmd}\"..." ${ORA_CRS_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else if [[ "~`grep '^Verification of system requirement was successful. $' ${_TmpFile}`~" = "~~" ]] then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} done # #---------------------------------------------------------------------------- # Use the "srvctl" utility to check status of the local database instance... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_srvctl_i_$$.out _Cmd="srvctl status instance -d ${DB_NAME} -i ${ORACLE_SID} -f -v" echo "\nRunning \"${_Cmd}\"..." ${ORACLE_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else if grep -v "is running on node" ${_TmpFile} then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Use the "srvctl" utility to check status of the entire database across the # cluster... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_srvctl_d_$$.out _Cmd="srvctl status database -d ${DB_NAME} -f -v" echo "\nRunning \"${_Cmd}\"..." ${ORACLE_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else if grep -v "is running on node" ${_TmpFile} then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Use the "olsnodes" and "srvctl" utilities to check status of the "nodeapps" # on all the nodes in the cluster... #---------------------------------------------------------------------------- for _n in `$CRS_HOME/bin/olsnodes` do _TmpFile=/tmp/${_Prgm}_srvctl_n_${_n}_$$.out _Cmd="srvctl status nodeapps -n ${_n}" echo "\nRunning \"${_Cmd}\"..." ${ORACLE_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else if grep -v "is running on node" ${_TmpFile} then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi fi rm -f ${_TmpFile} done # #---------------------------------------------------------------------------- # Extract the IP hostname or IP address of the VIP from "ocrdump" output... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_ocrdump_$$.out _DmpFile=/tmp/${_Prgm}_ocrdump_$$.dmp _Cmd="ocrdump ${_DmpFile}" echo "\nRunning \"${_Cmd}\"..." ${ORA_CRS_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else # #-------------------------------------------------------------------- # Parse out the "DATABASE.NODEAPPS..VIP.IP value from the # "ocrdump" output for the local node... #-------------------------------------------------------------------- _OcrDumpStr="DATABASE.NODEAPPS.${_HostName}.VIP.IP" echo "\nValidating \"ocrdump\" output for \"${_OcrDumpStr}\"..." _VIP=`awk \ '{if ($1=="['${_OcrDumpStr}']") \ {flag=1;} \ if (flag==1) \ if ($1=="ORATEXT") \ {print $3; flag=0}}' ${_DmpFile}` if [[ "~${_VIP}~" = "~~" ]] then echo "\tUnable to find/parse \"${_OcrDumpStr}\" value from dump" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tFound local VIP \"${_VIP}\" - OK" _Cmd="ping ${_VIP} -c 1" echo "\nRunning \"${_Cmd}\"..." # #------------------------------------------------------------ # if the VIP value is a hostname (not an IP address), then # use the "ping" utility to translate it to an IP address... #------------------------------------------------------------ _IpAddr=`${_Cmd} 2> /dev/null | \ grep "^PING" | \ awk '{print $3}' | \ sed 's/(//' | \ sed 's/)//'` if [[ "~${_IpAddr}~" = "~~" ]] then echo "\tUnable to translate \"${_VIP}\" to an IP address using \"ping\"..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tFound local VIP IP address is \"${_IpAddr}\" - OK" # #---------------------------------------------------- # use "ifconfig -a" to determine if the local VIP # is "plumbed"... #---------------------------------------------------- if [[ "~`ifconfig -a | grep ${_IpAddr}`~" = "~~" ]] then echo "\t\"ifconfig -a\" shows VIP \"${_IpAddr}\" not plumbed..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tLocal VIP \"${_IpAddr}\" plumbed - OK" fi fi fi # #-------------------------------------------------------------------- # Parse out the "SYSTEM.css.node_names..nodenum" value from # the "ocrdump" output... #-------------------------------------------------------------------- _OcrDumpStr="SYSTEM.css.node_names.${_HostName}.nodenum" echo "\nValidating \"ocrdump\" output for \"${_OcrDumpStr}\"..." _NodeNbr=`awk \ '{if ($1=="['${_OcrDumpStr}']") \ {flag=1;} \ if (flag==1) \ if ($1=="UB4") \ {print $4; flag=0}}' ${_DmpFile}` if [[ "~${_NodeNbr}~" = "~~" ]] then echo "\tUnable to find/parse \"${_OcrDumpStr}\" value from dump" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tFound local CSS node_number \"${_NodeNbr}\" - OK" fi # #-------------------------------------------------------------------- # Parse out the "SYSTEM.css.node_numbers.node.nodename" value from # the "ocrdump" output for the local node... #-------------------------------------------------------------------- _OcrDumpStr="SYSTEM.css.node_numbers.node${_NodeNbr}.nodename" _PublicNodeName=`awk \ '{if ($1=="['${_OcrDumpStr}']") \ {flag=1;} \ if (flag==1) \ if ($1=="ORATEXT") \ {print $3; flag=0}}' ${_DmpFile}` if [[ "~${_PublicNodeName}~" = "~~" ]] then echo "\tUnable to find/parse \"${_OcrDumpStr}\" value from dump" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"$_OcrDumpStr\" value is \"${_PublicNodeName}\" - OK" # #------------------------------------------------------------ # Check to make sure that the "CSS public node name" matches # the "hostname -s" output... #------------------------------------------------------------ if [[ "${_PublicNodeName}" != "${_HostName}" ]] then echo "\tCSS \"${_OcrDumpStr}\" value \"${_PublicNodeName}\" does not match \"hostname -s\"..."" integer _ExitStatus=${_ExitStatus}+1 fi # #------------------------------------------------------------ # if the "CSS public node name" value is a hostname (not an # IP address), then use the "ping" utility to translate it # to an IP address... #------------------------------------------------------------ _Cmd="ping ${_PublicNodeName} -c 1" echo "\nRunning \"${_Cmd}\"..." _IpAddr=`${_Cmd} 2> /dev/null | \ grep "^PING" | \ awk '{print $3}' | \ sed 's/(//' | \ sed 's/)//'` if [[ "~${_IpAddr}~" = "~~" ]] then echo "\t\"${_Cmd}\" of CSS \"${_OcrDumpStr}\" failed..."" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"${_OcrDumpStr}\" value has IP address \"${_IpAddr}\" - OK" # #---------------------------------------------------- # use "ifconfig -a" to determine if the local VIP # is "plumbed"... #---------------------------------------------------- if [[ "~`ifconfig -a | grep ${_IpAddr}`~" = "~~" ]] then echo "\t\"ifconfig -a\" shows CSS \"${_OcrDumpStr}\" value \"${_IpAddr}\" not plumbed..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"${_OcrDumpStr}\" value \"${_IpAddr}\" plumbed - OK" fi fi fi # #-------------------------------------------------------------------- # Parse out the "SYSTEM.css.node_numbers.node.hostname" value from the # "ocrdump" output for the local node... #-------------------------------------------------------------------- _OcrDumpStr="SYSTEM.css.node_numbers.node${_NodeNbr}.hostname" _PublicHostName=`awk \ '{if ($1=="['${_OcrDumpStr}']") \ {flag=1;} \ if (flag==1) \ if ($1=="ORATEXT") \ {print $3; flag=0}}' ${_DmpFile}` if [[ "~${_PublicHostName}~" = "~~" ]] then echo "\tUnable to find/parse \"${_OcrDumpStr}\" value from dump" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"$_OcrDumpStr\" value is \"${_PublicHostName}\" - OK" # #------------------------------------------------------------ # Check to make sure that the "CSS public HOSTNAME" matches # the "hostname -s" output... #------------------------------------------------------------ if [[ "${_PublicHostName}" != "${_HostName}" ]] then echo "\tCSS \"${_OcrDumpStr}\" value \"${_PublicHostName}\" does not match \"hostname -s\"..."" integer _ExitStatus=${_ExitStatus}+1 fi # #------------------------------------------------------------ # if the "CSS public HOSTNAME" value is a hostname (not an # IP address), then use the "ping" utility to translate it # to an IP address... #------------------------------------------------------------ _Cmd="ping ${_PublicHostName} -c 1" echo "\nRunning \"${_Cmd}\"..." _IpAddr=`${_Cmd} 2> /dev/null | \ grep "^PING" | \ awk '{print $3}' | \ sed 's/(//' | \ sed 's/)//'` if [[ "~${_IpAddr}~" = "~~" ]] then echo "\t\"${_Cmd}\" of CSS \"${_OcrDumpStr}\" failed..."" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"${_OcrDumpStr}\" value has IP address \"${_IpAddr}\" - OK" # #---------------------------------------------------- # use "ifconfig -a" to determine if the local VIP # is "plumbed"... #---------------------------------------------------- if [[ "~`ifconfig -a | grep ${_IpAddr}`~" = "~~" ]] then echo "\t\"ifconfig -a\" shows CSS \"${_OcrDumpStr}\" value \"${_IpAddr}\" not plumbed..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"${_OcrDumpStr}\" value \"${_IpAddr}\" plumbed - OK" fi fi fi # #-------------------------------------------------------------------- # Parse out the "SYSTEM.css.node_numbers.node.privatename" value from the # "ocrdump" output for the local node... #-------------------------------------------------------------------- _OcrDumpStr="SYSTEM.css.node_numbers.node${_NodeNbr}.privatename" _PrivateHostName=`awk \ '{if ($1=="['${_OcrDumpStr}']") \ {flag=1;} \ if (flag==1) \ if ($1=="ORATEXT") \ {print $3; flag=0}}' ${_DmpFile}` if [[ "~${_PrivateHostName}~" = "~~" ]] then echo "\tUnable to find/parse \"${_OcrDumpStr}\" value from dump" integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"$_OcrDumpStr\" value is \"${_PrivateHostName}\" - OK" # #------------------------------------------------------------ # if the "CSS private HOSTNAME" value is a hostname (not an # IP address), then use the "ping" utility to translate it # to an IP address... #------------------------------------------------------------ _Cmd="ping ${_PrivateHostName} -c 1" echo "\nRunning \"${_Cmd}\"..." _IpAddr=`${_Cmd} 2> /dev/null | \ grep "^PING" | \ awk '{print $3}' | \ sed 's/(//' | \ sed 's/)//'` if [[ "~${_IpAddr}~" = "~~" ]] then echo "\t\"${_Cmd}\" of CSS \"${_OcrDumpStr}\" failed..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"${_OcrDumpStr}\" value has IP address \"${_IpAddr}\" - OK" # #---------------------------------------------------- # use "ifconfig -a" to determine if the local VIP # is "plumbed"... #---------------------------------------------------- if [[ "~`ifconfig -a | grep ${_IpAddr}`~" = "~~" ]] then echo "\t\"ifconfig -a\" shows CSS \"${_OcrDumpStr}\" value \"${_IpAddr}\" not plumbed..." integer _ExitStatus=${_ExitStatus}+1 else _Echo "\tCSS \"${_OcrDumpStr}\" value \"${_IpAddr}\" plumbed - OK" fi fi fi fi rm -f ${_TmpFile} ${_DmpFile} # #---------------------------------------------------------------------------- # Verify that TNS listener is responding to status requests... # # If the log file for the TNS listener has been modified recently, then # display the last 30 lines of the file... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_lsnrctl_$$.out typeset -u _LsnrName="LISTENER_${_HostName}" _Cmd="lsnrctl status ${_LsnrName}" echo "\nRunning \"${_Cmd}\"..." ${ORACLE_HOME}/bin/${_Cmd} 2>&1 > ${_TmpFile} if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Echo "\t\t\"lsnrctl status ${_LsnrName}\" executed successfully - OK" _Cat /tmp/${_Prgm}_lsnrctl_$$.out fi # #---------------------------------------------------------------------------- # Attempt to extract the name of the TNS Listener's log file from the output # of "lsnrctl status"... #---------------------------------------------------------------------------- _LsnrLog=`grep "Listener Log File" ${_TmpFile} | awk '{print $4}'` rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Verify that TNS listener is responding to pinged connection requests, using # the DB_NAME as the TNS "connect string"... #---------------------------------------------------------------------------- _TnsString="" _TmpFile=/tmp/${_Prgm}_tnsping_$$.out _Cmd="tnsping ${DB_NAME}" echo "\nRunning \"${_Cmd}\"..." ${ORACLE_HOME}/bin/${_Cmd} 2>&1 > ${_TmpFile} if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _TnsString="${DB_NAME}" _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Verify that TNS listener is responding to pinged connection requests, using # the ORACLE_SID as the TNS "connect string"... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_tnsping_$$.out _Cmd="tnsping ${ORACLE_SID}" echo "\nRunning \"${_Cmd}\"..." ${ORACLE_HOME}/bin/${_Cmd} 2>&1 > ${_TmpFile} if (( $? != 0 )) then echo "\t\"${_Cmd}\" failed:" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else if [[ "~${_TnsString}~" = "~~" ]] then _TnsString=${ORACLE_SID} fi _Cat ${_TmpFile} _Echo "\t\t\"${_Cmd}\" executed successfully - OK" fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Connect to the RAC database instance through SQL*Plus, using a bogus # username and password. The only valid response to this connection # attempt is ORA-01017... #---------------------------------------------------------------------------- if [[ "~${_TnsString}~" = "~~" ]] then _TnsString=${DB_NAME} fi _TmpFile=/tmp/${_Prgm}_sqlplus_$$.out echo "\nRunning \"sqlplus\" for up/down test..." ${ORACLE_HOME}/bin/sqlplus -s /nolog << __EOF__ > ${_TmpFile} 2>&1 whenever oserror exit success whenever sqlerror exit success connect kjasdfoiasdfljkasflkj/jklfsakjlfdsaiofdsajk@${_TnsString} exit failure rollback __EOF__ if ! grep -q "ORA-01017" ${_TmpFile} > /dev/null 2>&1 then echo "\t\"sqlplus un/pw@${_TnsString}\" failed" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 else _Cat ${_TmpFile} _Echo "\t\t\"sqlplus un/pw@${_TnsString}\" executed successfully - OK" fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Connect to the RAC database instance through SQL*Plus, query V$THREAD and # GV$INSTANCE to determine the status of the database and instances... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_sqlplus2_$$.out echo "\nRunning \"sqlplus\" to check thread/instance status..." ${ORACLE_HOME}/bin/sqlplus -s /nolog << __EOF__ > ${_TmpFile} 2>&1 whenever sqlerror exit 1 rollback whenever oserror exit 2 rollback connect / as sysdba set echo off feedback off timing off time off tab off set pagesize 100 linesize 130 trimout on trimspool on pause off col db_name format a10 heading "DB|Name" col instance_name format a13 truncate heading "Instance|Name" col t_status format a8 truncate heading "Thread|Status" col i_status format a8 truncate heading "Instance|Status" col host_name format a30 truncate heading "Host Name" select d.name db_name, i.instance_name, i.host_name, t.status t_status, i.status i_status from v\$thread t, gv\$instance i, v\$database d where i.instance_name = t.instance and t.enabled = 'PUBLIC'; exit success __EOF__ case $? in 0) ;; 1) echo "\tUnable to connect as \"/ as sysdba\" via SQL*Plus" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 ;; 2) echo "\tQuery V\$THREAD, GV\$INSTANCE, V\$DATABASE failed via SQL*Plus" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 ;; *) echo "\"sqlplus\" failed unexpectedly" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 ;; esac # #---------------------------------------------------------------------------- # Determine how many database instances aren't open... #---------------------------------------------------------------------------- integer _NbrInst=`grep ${DB_NAME} ${_TmpFile} | wc -l` integer _NbrInstOpen=`grep ${DB_NAME} ${_TmpFile} | grep 'OPEN[ ]*OPEN' | wc -l` if (( ${_NbrInst} != ${_NbrInstOpen} )) then echo "\tOf ${_NbrInst} instances, only ${_NbrInstOpen} are \"OPEN\"" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # Connect to the RAC database instance through SQL*Plus, query V$TABLESPACE # and V$DATAFILE/TEMPFILE as well as V$LOG/LOG_FILE to find any database # files in unusual status... #---------------------------------------------------------------------------- _TmpFile=/tmp/${_Prgm}_sqlplus3_$$.out echo "\nRunning \"sqlplus\" to check database file status..." ${ORACLE_HOME}/bin/sqlplus -s /nolog << __EOF__ > ${_TmpFile} 2>&1 set echo off feedback off timing off time off tab off whenever sqlerror exit 1 rollback whenever oserror exit 2 rollback connect / as sysdba break on ts_name on report set pagesize 100 feedback off linesize 130 trimout on trimspool on pause off col ts_name format a25 truncate heading "Tablespace Name" col f_name format a40 truncate heading "File Name" col f_status format a12 truncate heading "File Status" select t.name ts_name, f.name f_name, f.status f_status from v\$tablespace t, v\$datafile f where t.ts# = f.ts# and f.status not in ('ONLINE','SYSTEM') union all select t.name ts_name, f.name f_name, f.status f_status from v\$tablespace t, v\$tempfile f where t.ts# = f.ts# and f.status not in ('ONLINE') union all select 'ONLINE REDO - "' || t.instance || '"' ts_name, f.member f_name, f.status f_status from v\$log l, v\$logfile f, v\$thread t where l.group# = f.group# and t.thread# = l.thread# and f.status is not null order by 1, 2; __EOF__ case $? in 0) ;; 1) echo "\tUnable to connect as \"/ as sysdba\" via SQL*Plus" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 ;; 2) echo "\tQuery on V\$TABLESPACE/DATA/TEMPFILE, V\$THREAD/LOG/LOG_FILE failed via SQL*Plus" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 ;; *) echo "\"sqlplus\" failed unexpectedly" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 ;; esac # #---------------------------------------------------------------------------- # Determine how many database files are showing unusual status... #---------------------------------------------------------------------------- if [[ "~`cat ${_TmpFile}`~" != "~~" ]] then echo "\tThe following database files do not appear \"ONLINE\":" sed 's/^/\t/' ${_TmpFile} integer _ExitStatus=${_ExitStatus}+1 fi rm -f ${_TmpFile} # #---------------------------------------------------------------------------- # ...if any possible error conditions have been detected and the shell # variable "_ExitStatus" is non-zero, then display whether any important # CRS or RDBMS/RAC "log" or "trace" files have been updated recently (i.e. # within the number of minutes specified by the shell variable "_FileAgeInMins"... #---------------------------------------------------------------------------- _DisplayLogsTraces="FALSE" if (( ${_ExitStatus} > 0 )) then _DisplayLogsTraces="TRUE" fi # if [[ "${_VerboseMode}" = "TRUE" ]] then _DisplayLogsTraces="TRUE" fi # if [[ "${_DisplayLogsTraces}" = "TRUE" ]] then # #-------------------------------------------------------------------- # Check if "messages" file has been updated in the past 180 minutes... #-------------------------------------------------------------------- _Messages=/var/log/messages _Echo "\nChecking if \"${_Messages}\" modified in past ${_FileAgeInMins} minutes..." if [ -f ${_Messages} ] then _FileList="`find /var/log \ -name messages \ -type f \ -mmin -${_FileAgeInMins} \ -print 2> /dev/null`" if [[ "~${_FileList}~" != "~~" ]] then echo "\n\"/var/log/messages\" file updated in past ${_FileAgeInMins} minutes..." ls -ltr ${_FileList} | sed 's/^/\t/' fi fi # #-------------------------------------------------------------------- # Check if CRS "alert.log" has been updated in the past 180 minutes... #-------------------------------------------------------------------- _CrsAlertLog=${ORA_CRS_HOME}/log/${_HostName}/alert${_HostName}.log _Echo "\nChecking if \"${_CrsAlertLog}\" modified in past ${_FileAgeInMins} minutes..." if [ -f ${_CrsAlertLog} ] then _FileList="`find ${ORA_CRS_HOME}/log/${_HostName} \ -name alert${_HostName}.log \ -type f \ -mmin -${_FileAgeInMins} \ -print 2> /dev/null`" if [[ "~${_FileList}~" != "~~" ]] then echo "\nCRS \"alert.log\" file updated in past ${_FileAgeInMins} minutes..." ls -ltr ${_FileList} | sed 's/^/\t/' _Echo "\nLast 30 lines in recently-updated \"alert.log\" file..." _Tail -30 ${_CrsAlertLog} fi fi # #-------------------------------------------------------------------- # Check if RDBMS "alert.log" has been updated in the past 180 # minutes... #-------------------------------------------------------------------- _AlertLog="" for _Dir in ${ORACLE_BASE}/admin \ ${ORACLE_HOME}/admin \ ${ORACLE_HOME}/rdbms/log do for _File in `find ${_Dir} \ -name "alert_${ORACLE_SID}.log" \ -type f \ -mmin -${_FileAgeInMins} \ -print 2> /dev/null` do if [[ "~${_AlertLog}~" = "~~" ]] then _AlertLog=${_File} echo "\nDatabase \"alert.log\" file updated in past ${_FileAgeInMins} minutes..." ls -ltr ${_AlertLog} | sed 's/^/\t/' _Echo "\nLast 30 lines in recently-updated \"alert.log\" file..." _Tail -30 ${_AlertLog} fi done done # #-------------------------------------------------------------------- # Check if TNS "listener.log" has been updated in the past 180 # minutes... #-------------------------------------------------------------------- if [[ "~${_LsnrLog}~" = "~~" ]] then _LsnrLog=${ORACLE_HOME}/network/log/listener_${_HostName}.log fi if [ -r ${_LsnrLog} ] then _LsnrLogDir=`dirname ${_LsnrLog}` _LsnrLogFile=`basename ${_LsnrLog}` _FileList="`find ${_LsnrLogDir} \ -name ${_LsnrLogFile} \ -type f \ -mmin -${_FileAgeInMins} \ -print 2> /dev/null`" if [[ "~${_FileList}~" != "~~" ]] then echo "\nTNS \"${_LsnrLogFile}\" file updated in past ${_FileAgeInMins} minutes..." ls -ltr ${_LsnrLog} | sed 's/^/\t/' _Echo "\nLast 30 lines in recently-updated \"${_LsnrLogFile}\" file..." _Tail -30 ${_LsnrLog} fi else echo "\tTNS Listener log \"${_LsnrLog}\" not readable..." integer _ExitStatus=${_ExitStatus}+1 fi # #-------------------------------------------------------------------- # Check for files in CRS-specific directories which have been # modified in the past 180 minutes... #-------------------------------------------------------------------- _PrevDisplayedDir="#" for _Dir in ${ORA_CRS_HOME}/log/${_HostName}/cssd \ ${ORA_CRS_HOME}/log/${_HostName}/crsd \ ${ORA_CRS_HOME}/log/${_HostName}/evmd \ ${ORA_CRS_HOME}/log/${_HostName}/client \ ${ORA_CRS_HOME}/log/${_HostName}/racg \ ${ORA_CRS_HOME}/log/${_HostName}/admin \ ${ORA_CRS_HOME}/crs/log \ ${ORA_CRS_HOME}/css/log \ ${ORA_CRS_HOME}/cv/log \ ${ORA_CRS_HOME}/evm/log \ ${ORA_CRS_HOME}/racg/log \ ${ORA_CRS_HOME}/srvm/log \ ${ORA_CRS_HOME}/opnm/logs \ ${ORACLE_HOME}/log/${_HostName}/racg \ ${ORACLE_HOME}/log/${_HostName}/client \ ${ORACLE_BASE}/admin/${ORACLE_SID}/log/udump \ ${ORACLE_BASE}/admin/${ORACLE_SID}/log/bdump do _Echo "\nChecking if any files in directory \"${_Dir}\" modified in past ${_FileAgeInMins} minutes..." if [ -d ${_Dir} ] then _FileList="`find ${_Dir} -mmin -${_FileAgeInMins} -type f -print 2> /dev/null`" if [[ "~${_FileList}~" != "~~" ]] then if [[ "${_Dir}" != "${_PrevDisplayedDir}" ]] then echo "\n...\"${_Dir}\" files modified in past ${_FileAgeInMins} minutes:" _PrevDisplayedDir=${_Dir} fi ls -ltr ${_FileList} | sed 's/^/\t/' # #-------------------------------------------- # if we're looking at EVM log files, then use # the "evmshow" utility to display the info... #-------------------------------------------- if [[ "${_Dir}" = "${ORA_CRS_HOME}/evm/log" ]] then for _File in ${_FileList} do _TmpFile=/tmp/${_Prgm}_evmshow_$$.out _Cmd="evmshow -t \"@timestamp @@\" ${_File}" echo "\nRunning \"${_Cmd}\"..." ${ORA_CRS_HOME}/bin/${_Cmd} > ${_TmpFile} 2>&1 if (( $? == 0 )) then sed 's/^/\t/' ${_TmpFile} fi rm -f ${_TmpFile} done fi fi fi done # fi # #---------------------------------------------------------------------------- # Offer a suggestion for gathering diagnostic information in the event an # SR needs to be created with Oracle Support... #---------------------------------------------------------------------------- if (( ${_ExitStatus} > 0 )) then echo "\nRun the CRS diagnostic collection script \"\$ORA_CRS_HOME/bin/diagcollection.pl\"" echo "to upload to MetaLink if you open an SR...\n" fi # #---------------------------------------------------------------------------- # Return exit status upon exit... #---------------------------------------------------------------------------- echo "\nDiagnostic script \"${_Prgm}.sh\" complete: `date`\n" exit ${_ExitStatus}