How to Start/Stop CRS

As root - set enviornment ( . oraenv and at prompt enter grid and press enter)
# cd /ora01/grid/11.2.0.4/grid/bin
# . ./oraenv
Type in grid

To stop Grid
# crsctl stop crs (will do one node at a time)
# ./crs_stop -all   (this will shutdown grid on both nodes, use only if directed by DBA)

To start grid
# ./crsctl start crs

To disable grid autostart
# crsctl disable crs  # to disable

To enable grid autostart(whenever we need to enable and start, the sequence is first enable then start)
# crsctl enable crs   # to enable

To check the grid status
# crsctl config crs

###FOR STANDALONE ORACLE SERVERS###

crsctl disable has
crsctl config has
crsctl enable has
crs_stat -t -v

to stop
crsctl stop has -f

to start
crsctl start has

11gR2 Clusterware and Grid Home - What You Need to Know

1050908.1 - troubleshoot grid startup issue
http://oracledbazone.wordpress.com/2012/07/17/useful-11gr2-rac-commands/
11gR2 Clusterware and Grid Home - What You Need to Know (Doc ID 1053147.1)


-- check  cluster name
   export GRID_HOME=/ora01/grid/11.2.0.2/grid
   export DB_HOME=/ora01/oracle/product/11.2.0.2/db
 
   $GRID_HOME/bin/cemutlo -n
 
--  Check Clusterware Resources status
   ps -ef |egrep "crsd.bin|ocssd.bin|evmd.bin|oprocd"
   ps -ef|grep d.bin
   ps -ef | egrep 'init|d.bin'
 
   cluvfy comp -list
 
   crsctl check crs
   $GRID_HOME/bin/crsctl stat res -t                 #all
   $GRID_HOME/bin/crsctl stat res -t -init           #only initialize
   $GRID_HOME/bin/crsctl stat res -p                 #long format
   $GRID_HOME/bin/crsctl query css votedisk
   $GRID_HOME/bin/ocrcheck
   $GRID_HOME/bin/oifcfg getif
   $GRID_HOME/bin/crsctl check crs
   crsctl check has
   crsctl query crs activeversion
   crsctl check cluster -all
   crs_stat -t -v


--  To find out config  information
     srvctl config network
     srvctl config nodeapps
     srvctl config asm
     srvctl config listener
     srvctl config scan
     srvctl config scan_listener
     $GRID_HOME/bin/srvctl config nodeapps -a
     $GRID_HOME/bin/srvctl config scan
     $GRID_HOME/bin/srvctl config network
     $GRID_HOME/bin/srvctl config asm -a
     $GRID_HOME/bin/srvctl config listener -l <listener-name> -a
     $DB_HOME/bin/srvctl config database -d <dbname> -a
     $DB_HOME/bin/srvctl config service -d <dbname> -s <service-name> -v

--  To find out non-grid status
     srvctl status nodeapps
     srvctl status asm
     srvctl status listener
     srvctl status scan
     srvctl status scan_listener
crs_stat -t -v
 
-- to start all resource (CRS, ASM , Listener and Databases)
   cd /ora01/grid/11.2.0.2/grid/bin
   ./crsctl start resource -all
   ./crsctl stop has ( will stop on node1 and move vip to node2)
   ./crs_stop -all ( will stop on both nodes)
   ./crsctl stop cluster -n node_name ( will stop asm, crs)
   ./crsctl stop crs ( will stop asm, crs)
 
   srvctl start cvu
   srvctl start oc4j
   srvctl status cvu
   alternate command
   crsctl start resource ora.cvu
   crsctl status resource ora.cvu



 -- start / stop non-rac
   # appsdbaworkshop.blogspot.com

    crsctl 11.2.0.2 to start/stop/check on standalone node

    crsctl config has
    crsctl stop has
    crsctl start has
    crsctl status resource
    crs_stop -all
    crs_start -all
 
   -- start resource manually
   crsctl start resource ora.cvu -n <nodename>
   crsctl start res ora.crsd -init
 
-- check network

  1. Find out Subnet ID from oifcfg from grid

    cd /ora01/grid/11.2.0.2/grid/bin
   
    oifcfg iflist -p -n
    1st col - network adapter name
    2nd col - subnet id
    3rd col - public/private
    4th col - netmask

  2. from root
     # ifconfig -a

  3. Subnet Info in Oracle Clusterware - OCR
     cd /ora01/grid/11.2.0.2/grid/bin
     oifcfg getif

-- check cluster nodes
   olsnodes -n
   olsnodes -c
   olsnodes -i  -s -n

-- OCR
   ocrcheck

-- Check voting disk
   crsctl query css votedisk

- check VIP
   srvctl status nodeapps
   srvctl start nodeapps
   crsctl stat res -t

-- check SCAN
   cluvfy comp scan -verbose
   srvctl config scan
   srvctl status scan

 
   srvctl config scan_listener
   srvctl status scan_listener
   lsnrctl status LISTENER_SCAN1
   lsnrctl service LISTENER_SCAN
   show parameter remote_listener
 
   crsctl stat res -w "TYPE = ora.scan_listener.type"
 
-- check listener
   srvctl status listener

-- check HAIP
   crsctl stat res -init -w "TYPE = ora.haip.type"
   oifcfg iflist -p -n
     will show   "bond1  169.254.0.0  UNKNOWN  255.255.0.0" for HAIP
 
   col host_name format a10
   sqlplus
     SELECT a.host_name, a.instance_name, b.name, b.ip_address
     FROM gv$cluster_interconnects b, gv$instance a
     WHERE a.inst_id=b.inst_id
     ORDER BY 1,2,3;
 
-- collect config
$GRID_HOME/bin/crsctl stat res -t
$GRID_HOME/bin/crsctl stat res -p
$GRID_HOME/bin/crsctl query css votedisk
$GRID_HOME/bin/ocrcheck
$GRID_HOME/bin/oifcfg getif
$GRID_HOME/bin/srvctl config nodeapps -a
$GRID_HOME/bin/srvctl config scan
$GRID_HOME/bin/srvctl config asm -a
$GRID_HOME/bin/srvctl config listener -l <listener-name> -a
$DB_HOME/bin/srvctl config database -d <dbname> -a
$DB_HOME/bin/srvctl config service -d <dbname> -s <service-name> -v

 
-- steps to  stop ( asm, instance, nodeapps, in 11gr2)
   $ srvctl stop instance -d RACDB -n racnode1
   $ srvctl stop vip -n racnode1 -f

--  check if the CRS/OHAS & services are enabled to autostart
   # $GRID_HOME/bin/crsctl config crs
   if not then enable
   # $GRID_HOME/bin/crsctl enable crs

--  shutdown crs services
   # $GRID_HOME/bin/crsctl stop crs
 
-- verify
   $GRID_HOME/bin/crsctl status resource -t

-- Change IP Interconnect address

-- Change VIP


-- get database name
   $ORACLE_HOME/bin/srvctl status home -o $ORACLE_HOME -s /tmp/oracle_home.stat

-- start databases using new command
   $ORACLE_HOME/bin/srvctl start home -o $ORACLE_HOME -s /tmp/oracle_home.stat

-- stop databases
   $ORACLE_HOME/bin/srvctl stop home -o $ORACLE_HOME -s /tmp/oracle_home.stat


  # http://surachartopun.com/2009/10/how-to-create-oracle-service-on-11gr2.html

  lnx20012@mzmd:/ora01/oracle $ cat /opt/oracle/admin/scripts/gridstat
     #!/usr/bin/ksh
     #
     # Check 11G RAC database Up/Down Status script
     #
     # Description:
     #    - Returns formatted version of crs_stat -t, in tabular
     #      format, with the complete rsc names and filtering keywords
     #   - The argument, $RSC_KEY, is optional and if passed to the script, will
     #     limit the output to HA resources whose names match $RSC_KEY.
     # Requirements:
     #   - $ORA_CRS_HOME should be set in your environment
     # Modifications
     #  RLM 5/2010 - use oraenv to set the environment, grid must be in oratab
     #             - check if search argument is null and substitute .* to create a valid syntax for Solaris
     ORACLE_SID=grid;ORAENV_ASK=NO;. oraenv;ORAENV_ASK=YES
     # If search argument is null, substitute .*, meaning match anything
     if [[ -z $1 ]]; then
       RSC_KEY='.*'
     else
       RSC_KEY=$1
     fi
     #QSTAT=-u
     QSTAT=-l
     AWK=/bin/awk
     $ORACLE_HOME/bin/crs_stat $QSTAT | $AWK \
       'BEGIN {printf "%-45s %-10s %-18s\n", "HA Resource", "Target", "State";
               printf "%-45s %-10s %-18s\n", "-----------", "------", "-----";
               FS="="; state = 0 }
       $1~/NAME/ && $2~/'$RSC_KEY'/ { appname=$2; state=1}
       state == 0 {next}
       $1~/TARGET/ && state == 1 {apptarget = $2; state=2}
       $1~/STATE/ && state == 2 {appstate = $2; state=3}
       state == 3 {printf "%-45s %-10s %-18s\n", appname, apptarget, appstate; state=0}'

Check the status of the cluster

Check the status of the cluster

[oracle@rac1 ~]$ crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

[oracle@rac1 ~]$ crsctl check cluster
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

[oracle@rac1 ~]$ crsctl check cluster -all
**************************************************************
rac1:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************

Check the status of the services on the cluster

[oracle@rac1 ~]$ crsctl status resource -t
——————————————————————————–
NAME TARGET STATE SERVER STATE_DETAILS
——————————————————————————–
Local Resources
——————————————————————————–
ora.DATA.dg
ONLINE ONLINE rac1
ora.LISTENER.lsnr
ONLINE ONLINE rac1
ora.asm
ONLINE ONLINE rac1
ora.gsd
OFFLINE OFFLINE rac1
ora.net1.network
ONLINE ONLINE rac1
ora.ons
ONLINE ONLINE rac1
ora.registry.acfs
ONLINE ONLINE rac1
——————————————————————————–
Cluster Resources
——————————————————————————–
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rac1
ora.cvu
1 ONLINE ONLINE rac1
ora.oc4j
1 ONLINE ONLINE rac1
ora.rac1.vip
1 ONLINE ONLINE rac1
ora.rac2.vip
1 OFFLINE OFFLINE
ora.scan1.vip
1 ONLINE ONLINE rac1
ora.testrac.db
1 OFFLINE OFFLINE Instance Shutdown

Stop crs on all nodes (only one node up in this example)

[root@rac1 bin]# ./crsctl stop cluster -all
CRS-2673: Attempting to stop ‘ora.crsd’ on ‘rac1'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on ‘rac1'
CRS-2673: Attempting to stop ‘ora.registry.acfs’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.DATA.dg’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.LISTENER.lsnr’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.cvu’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.oc4j’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.LISTENER_SCAN1.lsnr’ on ‘rac1'
CRS-2677: Stop of ‘ora.LISTENER.lsnr’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.rac1.vip’ on ‘rac1'
CRS-2677: Stop of ‘ora.LISTENER_SCAN1.lsnr’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.scan1.vip’ on ‘rac1'
CRS-2677: Stop of ‘ora.scan1.vip’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.rac1.vip’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.cvu’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.registry.acfs’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.oc4j’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.DATA.dg’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.asm’ on ‘rac1'
CRS-2677: Stop of ‘ora.asm’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.ons’ on ‘rac1'
CRS-2677: Stop of ‘ora.ons’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.net1.network’ on ‘rac1'
CRS-2677: Stop of ‘ora.net1.network’ on ‘rac1' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on ‘rac1' has completed
CRS-2677: Stop of ‘ora.crsd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.ctssd’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.evmd’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.asm’ on ‘rac1'
CRS-2677: Stop of ‘ora.asm’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.cluster_interconnect.haip’ on ‘rac1'
CRS-2677: Stop of ‘ora.evmd’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.cluster_interconnect.haip’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.ctssd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.cssd’ on ‘rac1'
CRS-2677: Stop of ‘ora.cssd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.diskmon’ on ‘rac1'
CRS-2677: Stop of ‘ora.diskmon’ on ‘rac1' succeeded

Stop crs on specifc node

[root@rac1 bin]# ./crsctl stop has
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on ‘rac1'
CRS-2673: Attempting to stop ‘ora.crsd’ on ‘rac1'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on ‘rac1'
CRS-2673: Attempting to stop ‘ora.cvu’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.LISTENER.lsnr’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.registry.acfs’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.DATA.dg’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.LISTENER_SCAN1.lsnr’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.oc4j’ on ‘rac1'
CRS-2677: Stop of ‘ora.LISTENER.lsnr’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.rac1.vip’ on ‘rac1'
CRS-2677: Stop of ‘ora.rac1.vip’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.LISTENER_SCAN1.lsnr’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.scan1.vip’ on ‘rac1'
CRS-2677: Stop of ‘ora.scan1.vip’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.cvu’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.oc4j’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.registry.acfs’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.DATA.dg’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.asm’ on ‘rac1'
CRS-2677: Stop of ‘ora.asm’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.ons’ on ‘rac1'
CRS-2677: Stop of ‘ora.ons’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.net1.network’ on ‘rac1'
CRS-2677: Stop of ‘ora.net1.network’ on ‘rac1' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on ‘rac1' has completed
CRS-2677: Stop of ‘ora.crsd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.drivers.acfs’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.mdnsd’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.crf’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.ctssd’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.evmd’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.asm’ on ‘rac1'
CRS-2677: Stop of ‘ora.asm’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.cluster_interconnect.haip’ on ‘rac1'
CRS-2677: Stop of ‘ora.evmd’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.crf’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.mdnsd’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.cluster_interconnect.haip’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.drivers.acfs’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.ctssd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.cssd’ on ‘rac1'
CRS-2677: Stop of ‘ora.cssd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.gipcd’ on ‘rac1'
CRS-2673: Attempting to stop ‘ora.diskmon’ on ‘rac1'
CRS-2677: Stop of ‘ora.gipcd’ on ‘rac1' succeeded
CRS-2673: Attempting to stop ‘ora.gpnpd’ on ‘rac1'
CRS-2677: Stop of ‘ora.diskmon’ on ‘rac1' succeeded
CRS-2677: Stop of ‘ora.gpnpd’ on ‘rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on ‘rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.

Disable CRS from starting on reboot

[root@rac1 bin]# ./crsctl disable has
CRS-4621: Oracle High Availability Services autostart is disabled.

Enable CRS to start on reboot

[root@rac1 bin]# ./crsctl enable has
CRS-4622: Oracle High Availability Services autostart is enabled.

Start crs on specifc node

[root@rac1 bin]# ./crsctl start has
CRS-4123: Oracle High Availability Services has been started.

Check database status

[oracle@rac1 ~]$ srvctl status database -d RACDB
Instance RACDB1 is running on node rac1
Instance RACDB2 is running on node rac2

Stop an instance

[oracle@rac1 ~]$ srvctl stop instance -i RACDB1 -d RACDB
[oracle@rac1 ~]$ srvctl status database -d RACDB
Instance RACDB1 is not running on node rac1
Instance RACDB2 is running on node rac2

Start an instance

[oracle@rac1 ~]$ srvctl start instance -i RACDB1 -d RACDB
[oracle@rac1 ~]$ srvctl status database -d RACDB
Instance RACDB1 is running on node rac1
Instance RACDB2 is running on node rac2

Stop the database

[oracle@rac1 ~]$ srvctl stop database -d RACDB
[oracle@rac1 ~]$ srvctl status database -d RACDB
Instance RACDB1 is not running on node rac1
Instance RACDB2 is not running on node rac2

Start the database

[oracle@rac1 ~]$ srvctl start database -d RACDB
[oracle@rac1 ~]$ srvctl status database -d RACDB
Instance RACDB1 is running on node rac1
Instance RACDB2 is running on node rac2

Stop the listener on a Node

[oracle@rac1 ~]$ srvctl status listener -n rac1
Listener LISTENER is enabled on node(s): rac1
Listener LISTENER is not running on node(s): rac1

Start the listener on a Node

[oracle@rac1 ~]$ srvctl status listener -n rac1
Listener LISTENER is enabled on node(s): rac1
Listener LISTENER is running on node(s): rac1



Check the voting disk

[oracle@rac1 ~]$ crsctl query css votedisk
## STATE File Universal Id File Name Disk group
– —– —————– ——— ———
1. ONLINE 91b18024f9674f83bf5117d3c0996a05 (ORCL:VOL1) [DATA]
Located 1 voting disk(s).

Query the network addresses

[oracle@rac1 ~]$ oifcfg getif
eth0 192.168.245.0 global public
eth1 192.168.126.0 global cluster_interconnect

Show OCR backup – the voting file (disk) is also backed up with the OCR backup

[root@rac1 bin]# ./ocrconfig -showbackup

rac1 2012/07/17 07:47:17 /u01/app/11.2.0/grid/cdata/rac-cluster/backup00.ocr

rac1 2012/07/17 03:47:15 /u01/app/11.2.0/grid/cdata/rac-cluster/backup01.ocr

rac1 2012/07/16 23:47:13 /u01/app/11.2.0/grid/cdata/rac-cluster/backup02.ocr

rac1 2012/07/16 19:47:11 /u01/app/11.2.0/grid/cdata/rac-cluster/day.ocr

rac1 2012/07/16 19:47:11 /u01/app/11.2.0/grid/cdata/rac-cluster/week.ocr

rac1 2012/03/23 11:30:19 /u01/app/11.2.0/grid/cdata/rac-cluster/backup_20120323_113019.ocr

Take a manual backup of the OCR

[root@rac1 bin]# ./ocrconfig -manualbackup

rac1 2012/07/17 10:08:28 /u01/app/11.2.0/grid/cdata/rac-cluster/backup_20120717_100828.ocr

rac1 2012/07/17 10:07:39 /u01/app/11.2.0/grid/cdata/rac-cluster/backup_20120717_100739.ocr

rac1 2012/03/23 11:30:19 /u01/app/11.2.0/grid/cdata/rac-cluster/backup_20120323_113019.ocr

Basics of 10g RAC/CRS

IMP: login as root for startup and shutdown

To Start CRS
Go to $CRS_HOME/bin
# ./crsctl start crs

Or

#  /etc/init.d/init.crs start

Check CRSD process

ps -ef|grep crsd.bin

To Stop CRS
Go to $CRS_HOME/bin
# ./crsctl stop crs

Or

# /etc/init.d/init.crs stop

CRS enable/disable

Go to $CRS_HOME/bin
crsctl enable crs

crsctl disable crs # ensures that the services do not come up after a reboot.

To find out the value of your CLUSTER_NAME from CRS (OCR),

  cd $CRS_HOME/bin

  ./cemutlo -n

lists the version of CRS software installed

crsctl query crs softwareversion []

lists the CRS software operating version

crsctl query crs activeversion

Status of CRS services
Go to $CRS_HOME/bin
bash-3.00$ ./crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.cmwdb1.gsd application ONLINE ONLINE cmwdb1
ora.cmwdb1.ons application ONLINE ONLINE cmwdb1
ora.cmwdb1.vip application ONLINE ONLINE cmwdb1
ora.cmwdb2.gsd application ONLINE ONLINE cmwdb2
ora.cmwdb2.ons application ONLINE ONLINE cmwdb2
ora.cmwdb2.vip application ONLINE ONLINE cmwdb2


Check current OCR file

bash-3.00$ ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 2
Total space (kbytes) : 102188
Used space (kbytes) : 2140
Available space (kbytes) : 100048
ID : 617463612
Device/File Name : /dev/vx/rdsk/racdg/rac_srvconfig
Device/File integrity check succeeded

Device/File not configured

Cluster registry integrity check succeeded


Check current CRS file

bash-3.00$ crsctl query css votedisk
0. 0 /dev/vx/rdsk/racdg/crs

located 1 votedisk(s).

Files need to backed for CRS

CRS HOME and startup scripts

$CRS_HOME
/etc/init.d/init.cssd
/etc/init.d/init.crs
/etc/init.d/init.crsd
/etc/init.d/init.evmd
/var/opt/oracle
/etc/inittab

OCR file

bash-3.00$ ocrconfig -showbackup
cmwdb1 2010/09/27 12:17:02 /oracle/crs/cdata/crs
cmwdb1 2010/09/27 08:17:01 /oracle/crs/cdata/crs
cmwdb1 2010/09/27 04:17:00 /oracle/crs/cdata/crs
cmwdb1 2010/09/26 12:16:57 /oracle/crs/cdata/crs
cmwdb1 2010/09/19 08:16:08 /oracle/crs/cdata/crs

CRS file
bash-3.00$ crsctl query css votedisk
0. 0 /dev/vx/rdsk/racdg/crs
dd if=/dev/vx/rdsk/racdg/crs of=/oracle/crs/data/crs/crs_backup.dmp bs=4k

CRS Log files location
$CRS_HOME/log/hostname

cluster verification utlity

     cluvfy comp crs -n all -verbose

     cluvfy comp ocr -n all -verbose

     cluvfy comp nodecon -n all -verbose

     olsnodes

     cluvfy comp -list

     cluvfy stage -list

     cvuqdisk

     cluvfy stage -post crsinst -n all -verbose  # as oracle

Do you want to check the health of the Clusterware?

     crsctl check crs

     crsctl check evmd

     crsctl check cssd

     crsctl check crsd

           

# OEM User Defined OS Metric.

    Script is called by OEM to monitor the health of CRSd on a particular node.

    # UserDefinedHostScripts/checkCRS.sh

    The script runs the "crsctl check crsd" command. The normal output

    is "CRS appears healthy". Any other response will trigger an alert.

    The alert is configured to send an email and an snmp trap to Unicenter.

    #!/usr/bin/ksh

    if [[ $(/ora01/oracle/product/crs_1020/bin/crsctl check crsd) = "CRS appears healthy" ]]

      then print "em_result=0\nem_message=CRSd is healthy\n"

      else print "em_result=1\nem_message=CRSd not responding\n"

    fi

    exit

CRS reboot issue

Whenever a node is having issues joining the cluster back post reboot, here is a quick check list :

    * /var/log/messages

    * ifconfig

    * ip route

    * /etc/hosts

    * /etc/sysconfig/network-scripts/ifcfg-eth*

    * ethtool

    * mii-tool

    * cluvfy

    * $CRS_HOME/log/hostname

To find the root cause, please provide the following from all the nodes to Oracle Support.

1)- OSWatcher output from all the nodes

2)- All log files under $CRS_HOME/log. You can use diagcollection.pl to get all log files.

3)- System OS logs "/var/log/messages".

4)- OPROCD log files under /etc/oracle/oprocd or /var/opt/oracle/oprocd.

5) Capture output of the cluvfy command.

Execute the following commands as oracle:

>ORACLE_SID=crs;ORAENV_ASK=NO;. oraenv;ORAENV_ASK=YES

>HOST=$(hostname)

>cd /opt/oracle/admin/SR

>script $HOST.cluvfy.log

>cluvfy stage -post crsinst -n all -verbose

>exit

Upload the files to Oracle in support of your SR.

Using the diagcollection.pl procedure

Run the following commands on each node. These commands must be run as root.

>export ORA_CRS_HOME=/ora01/oracle/product/crs_1020

>export ORACLE_BASE=/ora01/oracle

>export ORACLE_HOME=/ora01/oracle/product/crs_1020

>export HOSTNAME=$(hostname)

>cd $ORA_CRS_HOME/bin

>script diag.log

>env

>id

>./diagcollection.pl -collect

>exit

This creates the

            diag.log file in $ORA_CRS_HOME/bin and

these files crsData_.tar.gz,

            ocrData_.tar.gz,

            oraData_.tar.gz and

            basData_.tar.gz

If you get a prompt to overwrite existing tar.gz files, go ahead and overwrite.

This is a perl script, so /bin/perl has to be in your PATH.

Hope this helps! Rupam

Register the database with CRS

CRS  SRVCTL

Register database with CRS to enable automatic startup/shutdown of database and services when server is bounced

To register the database with CRS

crs_stat in 10g

Check 10g Grid Status, shows RAC database Up/Down Status

#!/usr/bin/ksh

#

# Check 10G Grid status - RAC database Up/Down Status script

#

# Description:

#    - Returns formatted version of crs_stat -t, in tabular

#      format, with the complete rsc names and filtering keywords

#   - The argument, $RSC_KEY, is optional and if passed to the script, will

#     limit the output to HA resources whose names match $RSC_KEY.

# Requirements:

#   - $ORA_CRS_HOME should be set in your environment

# Modifications

#  7/2006 - use oraenv to set the environment, crs must be in oratab

#             - check if search argument is null and substitute .* to create a valid syntax for Solaris

ORACLE_SID=crs;ORAENV_ASK=NO;. oraenv;ORAENV_ASK=YES

# If search argument is null, substitute .*, meaning match anything

if [[ -z $1 ]]; then

  RSC_KEY='.*'

else

  RSC_KEY=$1

fi

QSTAT=-u

AWK=/bin/awk

$ORACLE_HOME/bin/crs_stat $QSTAT | $AWK \

  'BEGIN {printf "%-45s %-10s %-18s\n", "HA Resource", "Target", "State";

          printf "%-45s %-10s %-18s\n", "-----------", "------", "-----";

          FS="="; state = 0 }

  $1~/NAME/ && $2~/'$RSC_KEY'/ { appname=$2; state=1}

  state == 0 {next}

  $1~/TARGET/ && state == 1 {apptarget = $2; state=2}

  $1~/STATE/ && state == 2 {appstate = $2; state=3}

  state == 3 {printf "%-45s %-10s %-18s\n", appname, apptarget, appstate; state=0}'

crs_stat in 11g

Check 11g Grid Status, shows RAC database Up/Down Status

#!/usr/bin/ksh

#  works on linux

# Description:

# - Returns formatted version of crs_stat -t, in tabular

# format, with the complete rsc names and filtering keywords

# - The argument, $RSC_KEY, is optional and if passed to the script, will

# limit the output to HA resources whose names match $RSC_KEY.

# Requirements:

# - $ORA_CRS_HOME should be set in your environment

# Modifications

# 5/2010 - use oraenv to set the environment, grid must be in oratab

# - check if search argument is null and substitute .* to create a valid syntax for Solaris

ORACLE_SID=grid;ORAENV_ASK=NO;. oraenv;ORAENV_ASK=YES

# If search argument is null, substitute .*, meaning match anything

if [[ -z $1 ]]; then

RSC_KEY='.*'

else

RSC_KEY=$1

fi

#QSTAT=-u

QSTAT=-l

AWK=/bin/awk

$ORACLE_HOME/bin/crs_stat $QSTAT | $AWK \

'BEGIN {printf "%-45s %-10s %-18s\n", "HA Resource", "Target", "State";

printf "%-45s %-10s %-18s\n", "-----------", "------", "-----";

FS="="; state = 0 }

$1~/NAME/ && $2~/'$RSC_KEY'/ { appname=$2; state=1}

state == 0 {next}

$1~/TARGET/ && state == 1 {apptarget = $2; state=2}

$1~/STATE/ && state == 2 {appstate = $2; state=3}

state == 3 {printf "%-45s %-10s %-18s\n", appname, apptarget, appstate; state=0}'

CRS ( Oracle Clusterware ) Cheat Sheet

Oracle Clusterware enables servers in an Oracle database Real Application Cluster to coordinate simultaneous workload on the same database files. The crsctl command provides administrators many useful capabilities. For example, with crsctl, you can check Clusterware health disable/enable Oracle Clusterware startup on boot, find information on the voting disk and check the Clusterware version, and more.

1. Do you want to check the health of the Clusterware?
# crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy

if you receive any output other than the "…appears healthy" output, then kill crsd.bin on the node that gave you the unhealthy output ( login as root)

2. Do you want to reboot a node for maintenance without Clusterware coming up on boot?
## Disable clusterware on machine2 bootup:
# crsctl disable crs
## Stop the database then stop clusterware processes:
# srvctl stop instance –d db –i db2
# crsctl stop crs
# reboot

## Enable clusterware on machine bootup:
# crsctl enable crs
# crsctl start crs
# srvctl start instance –d db –i db2

3. Do you wonder where your voting disk is?
# crsctl query css votedisk
0. 0 /dev/raw/raw2

4. Do you need to find out what clusterware version is running on a server?
# crsctl query crs softwareversion
CRS software version on node [db2] is [10.2.0.2.0]

STOP CRS

Connect as root and from ORA_CRS_HOME execute the following command on both servers

[root@vmractest1 bin]# ./crsctl stop crs
Stopping resources.
Successfully stopped CRS resources
Stopping CSSD.
Shutting down CSS daemon.
Shutdown request successfully issued.

Restart CRS on Both Nodes

[root@vmractest1 bin]# ./crsctl start crs
[root@vmractest2 bin]# ./crsctl start crs
Restarting CRS brought up all resources:


CRS Verion

$ crsctl query crs softwareversion
$ crsctl query crs activeversion

Monitor CRS Heath using OEM

$ cat /opt/oracle/admin/OEM/UserDefinedHostScripts/checkCRS.sh
#!/usr/bin/ksh
# OEM User Defined OS Metric. Script is called by OEM to monitor the health of CRSd on a particular node.
if [[ $(/ora01/oracle/product/crs_1020/bin/crsctl check crsd) = "CRS appears healthy" ]]
then print "em_result=0\nem_message=CRSd is healthy\n"
else print "em_result=1\nem_message=CRSd not responding\n"
fi
exit