============================================================
Clustering
============================================================
Version: 2017.03.02.0001
============================================================

################################################################################
#ServiceGuard


	#Modify the cluster file
	#GUI - browser
		service hpsmhd start

		http://<machine_name>.<domain_name>:2301
	
		top-menu-row in browser:   Tools -> Serviceguard Manager

	#Modify the cluster file
	#get ENV variables: i.e. $SGCONF  $SGLOG
	 . /etc/cmcluster.conf

	#file that contains variables to major directories (source this file)
		/etc/cmcluster.conf

	#Log file:
		/usr/local/cmcluster/run/log/$PKGNAME.log

		or

		/etc/cmcluster/$PKGNAME/$PKGNAME.ctrl.log

    #many paths to this cm - it lives in too many places
    	MYCMD1="cmviewcl"
	#old version, works still
    	MYCMD1OPTS="-l package"
	#new version
    	#MYCMD1OPTS="-f line -v"
    for MYPATH in /opt/cmcluster/bin /usr/sbin /usr/local/cmcluster/bin
    do
        if [[ -e $MYPATH/cmviewcl ]]
        then
            MYCMD1="$MYPATH/cmviewcl"
            break
        fi
    done

	#cluster config: /etc/cmcluster/cluster.conf
	#pkg config: /etc/cmcluster/PKG/PKG.conf
	#pkg control: /etc/cmcluster/PKG/PKG.ctrl

	#cmviewconf is depracted as of A.11.19, use cmviewcl instead
	cmviewcl -v  
	cmviewcl -v -f line | grep vg
	cmviewcl -v -f line -p $MYPKGNAME  | grep vg
	cmquerycl -v

   . /etc/cmcluster.conf

   cmviewcl -v | tee /tmp/cmviewcl.$(date +%Y%m%d)

   cmquerycl -v

   cmgetconf $SGCONF/verify.ascii
   cmcheckconf -v -C $SGCONF/verify.ascii

   cmquerycl -v -C $SGCONF/verify.ascii [-n $nodex [-n nodey [-n nodez...]] ] \
            | tee /tmp/cluster_check.txt

   PKGNAMES="$(cmviewcl -l package \
            | grep -v ^$ \
            | grep -v STATE \
            | awk '{print $1}'\
            | tr '\012' ' ')"

   #See if each package is on its primary node
   for PKGNAME in $PKGNAMES
   do
      echo $PKGNAME | tee -a /tmp/cmviewcl_pkgname_current
      cmviewcl -v -p $PKGNAME | grep current | tee -a /tmp/cmviewcl_pkgname_current
   done


   #If there are no conf files, create them
   for PKGNAME in $PKGNAMES
   do
      echo $PKGNAME
      if [[ ! -d $SGCONF/$PKGNAME ]]
      then
         echo "INFO: Making a directory"
         mkdir -p $SGCONF/$PKGNAME
      fi
      if [[ -s $SGCONF/$PKGNAME/$PKGNAME.conf ]]
      then
         echo "INFO: Making a backup"
         cp $SGCONF/$PKGNAME/$PKGNAME.conf \
            $SGCONF/$PKGNAME/$PKGNAME.conf.$(date +%Y%m%d)

      fi
      echo "INFO: Saving the conf"
      cmgetconf -p $PKGNAME $SGCONF/$PKGNAME/$PKGNAME.conf
   done


   #Record NODE_NAME entries for each package
   for PKGNAME in $PKGNAMES
   do
      echo $PKGNAME
      grep ^NODE_NAME $SGCONF/$PKGNAME/$PKGNAME.conf
   done | tee -a /tmp/package_conf_names.out

   #compare files on each of the nodes for each PKG
   #run ON EACH SYSTEM in the cluster and compare the output
   for PKGNAME in $PKGNAMES
   do
      echo $SGCONF/$PKGNAME
      cd $SGCONF/$PKGNAME
      cksum $PKGNAME.conf $PKGNAME.c*trl
      cd -
   done

   tail /var/adm/syslog/syslog.log
	

	HPUX
		/var/adm/syslog/syslog.log
	OTHERS
		tail /var/log/messages
	
	#start the SG cluster processes manually on a node
	cmrunnode

	#stop  the SG cluster processes manually on a node
	cmhaltnode

	#start the SG cluster processes manually in a cluster
	cmruncl

	#stop  the SG cluster processes manually in a cluster
	cmhaltcl

	cmviewcl -l package #view status/state of pkg

	cmhaltpkg $PKGNAME #stop pkg

	cmrunpkg  $PKGNAME #start pkg
	cmrunpkg  -n $NODENAME $PKGNAME #start pkg

	#Use this to enable and AUTO_START enable a package
	cmmodpkg -e $PKG   #enable monitoring (unfreeze) for a pkg

	cmmodpkg -d $PKG   #diable monitoring (freeze)   for a pkg

	#To disable on a node
		cmmodpkg -d -n NODE  $PKG   #enable switching on NODE

	#If switching is disabled for a node
		cmmodpkg -e -n NODE  $PKG   #enable switching on NODE

	#failover
	cmhaltpkg $PKGNAME #stop pkg
	cmrunpkg  -n $NODENAME $PKGNAME #start pkg


	########################
	#START: verify your cluster
	########################
    #Service Guard

    . /etc/cmcluster.conf

    cmviewcl -v | tee /tmp/cmviewcl.$(date +%Y%m%d)
    cmquerycl -v
 
    cmquerycl -v -C $SGCONF/verify.ascii -n $nodex [-n nodey [-n nodez...]] \
             | tee /tmp/cluster_check.txt
 
    PKGNAMES="$(cmviewcl -l package \
             | grep -v ^$ \
             | grep -v STATE \
             | awk '{print $1}'\
             | tr '\012' ' ')"
 
    #See if each package is on its primary node
    for PKGNAME in $PKGNAMES
    do
       echo $PKGNAME
       cmviewcl -v -p $PKGNAME | grep current
    done


    #If there are not conf files, create them
    for PKGNAME in $PKGNAMES
    do
       echo $PKGNAME
       if [[ ! -d $SGCONF/$PKGNAME ]]
       then
          echo "INFO: Making a directory"
          mkdir -p $SGCONF/$PKGNAME
       fi
       if [[ -s $SGCONF/$PKGNAME/$PKGNAME.conf ]]
       then
          echo "INFO: Making a backup"
          cp $SGCONF/$PKGNAME/$PKGNAME.conf \
             $SGCONF/$PKGNAME/$PKGNAME.conf.$(date +%Y%m%d)
 
       fi
       echo "INFO: Saving the conf"
       cmgetconf -p $PKGNAME $SGCONF/$PKGNAME/$PKGNAME.conf
    done
 
 
    #Record NODE_NAME entries for each package
    for PKGNAME in $PKGNAMES
    do
       echo $PKGNAME
       grep ^NODE_NAME $SGCONF/$PKGNAME/$PKGNAME.conf
    done | tee -a /tmp/package_conf_names.out
 
    #compare files on each of the nodes for each PKG
    #run ON EACH SYSTEM in the cluster and compare the output
    for PKGNAME in $PKGNAMES
    do
       echo $PKGNAME
       cd $SGCONF/$PKGNAME
       cksum $PKGNAME.conf $PKGNAME.ctrl
    done

    cmcheckconf -v -C $SGCONF/verify.ascii
 
    tail /var/adm/syslog/syslog.log
 
    #Loop on each node, checking SYMDEV disk-ID and disk dev on each system
    mkdir -p /nfs/SVC
    mount xxx:/install/SVC /nfs/SVC
    /nfs/SVC/bin/show_new_disk.sh
    cd /; umount /nfs/SVC

	########################
	#END: verify your cluster
	########################

    ########################
    #START: IF SG clustered
    ########################
    #Change from "enabled/started" to "disabled/unmanaged"
    cmviewcl
 
 	#one server
    for PKGNAME in $PKGNAMES
    do
       echo $PKGNAME
       echo "cmmodpkg -d $PKGNAME"
       cmmodpkg -d $PKGNAME
    done

    cmviewcl


    for MYPKG in $PKGNAMES
    do
		NODES=$(cmviewcl -v -l node -p $MYPKG \
	   	      | awk '/NODE_TYPE/,/^      $/ {print $4}' \
				 | sed -e 1d \
				 | tr '\012' ' ')
		MYNODES=$(echo $NODES | sed -e 's/ / -n /g;s/.*/-n &/')
		echo $MYNODES
		cmviewcl -p $MYPKG | grep running
		#cmmodpkg -v -n node1 -n node2 -n node3 -d $MYPKG
		cmmodpkg -v $MYNODES -d $MYPKG
		cmmodpkg -e $MYPKG
		cmviewcl -p $MYPKG | grep running
    done

	#IF you are reinstalling, and the s/w says there is an existing config 
	#and won't remove the serviceguard pkg, then do a: 
		#rm /usr/local/cmcluster/conf/cmclconfig
	
	#To save off a current config, if you are upgrading:
		#1.	Backup the recent cluster configuration in a file
		# a.	cmgetconf  > cluster.ascii
		#2.	Backup all the packages’ s recent configuration in separate files 
		# a.	cmgetconf –p <package_name>  >  pkg_<package_name>.ascii
		#3.	Halt and Delete the cluster 
		# a.	cmhaltcl –f 
		# b.	cmdeleteconf 
		#4.	Remove SG/LX evaluation version on all nodes 
		#5.	Install production SG/LX version from DVD or through the image
		#   downloaded (i.e. E-Delivery) on all nodes. 
		#   You can use the cmeasyinstall command.
		#6.	Apply the cluster configuration file  first 
		# a.	cmcheckconf -C cluster.ascii
		# b.	cmapplyconf -C cluster.ascii
		#7.	Apply all the package configuration files 
		# a.	cmcheckconf -P pkg_<package name>.ascii
		# b.	cmapplyconf -P pkg_<package name>.ascii
	
	#see config: legacy (old) or modular (new)

		#Either use the "-f" or not (if available)

		cmviewcl -v -f line
		cmviewcl -v -f line | grep style
		cmviewcl -v -f line | grep script

	#To determine whether legacy packages are in use on the cluster
	cmviewcl -v -f line | grep style
		#package:SG-CFS-pkg|style=legacy
		#package:CFSdg_pkg|style=legacy
		#package:situxdg_pkg|style=legacy
		#package:cases_pkg|style=legacy
		#package:situx_pkg|style=legacy
		#package:dump|style=modular 

	#If it exists and runs, determine whether the paths used by the package
	#control scripts involves 
		#modular packages: /etc/cmcluster/scripts 
		#legacy-style package: /etc/cmcluster/<package_name>

	Notes:

	A.11.18 offers a legacy to modular package migration utility

	A.11.20 offers the ability to configure multiple CFS Mount Points 
		    and CVM Disk Groups in one modular package.

	#Migration tool
	cmmigratepkg

	#(my two cents)
	#IF you want to use legacy scripts, but have the new modular
	#functionality, then create another "layer" and have the modular script
	#to have a case statement that calls the start or stop based upon
	#the passed in argument.

    conf
        #user mondular (new)
			#
			# must have the following in the PKG.conf file:
			#
			#    module_name                     sg/external
			#    module_version                  1
			#
            #Example: $SGCONF/examples/external_script.template
			#
            external_pre_script   /etc/cmcluster/...
            #in order as defined on startup, reverse order on shutdown
               external_script       /etc/cmcluster/...first
               external_script       /etc/cmcluster/...second
               external_script       /etc/cmcluster/...third

            #take 3 args:
                #start
                #stop
                #validate
            #give 3 exits:
                #0 - success
                #1 - indicating the package will be halted, and
                #    shouldn't be restarted, as a result of failure
                #2 - indicating the package will be restarted on another
                #    node, or halted if not other node is available

        #use legacy
            #take 3 args: start, stop, validate
               _script /usr/cmcluster/scripts/up
               _script /usr/cmcluster/scripts/up1

#new vg in service guard cluster

    vchange -a n $MYVG
	vchange -c y $MYVG
	vchange -a e $MYVG


################################################################################
#HAE
    MYCMD2="/usr/sbin/crm"
    MYCMD2OPTS="resource list"

	crm status
	crm_mon -1
	crm_mon -1nf
	crm resource status [resource_name]
	crm configure show [xml]
	crm_verify -VL

	cat /var/lib/hearbeat/crm/cib.xml
	service heartbeat status

	crm status
			#Connection to cluster failed: connection failed

	/etc/init.d/openais start

	crm status
		#Node xxxX: UNCLEAN (offline)
		#Node xxxY: UNCLEAN (offline)

		#status is that node is unclean
			
			crm node clearstate <node_name> 

		#in case the cluster will not start resource groups
		#crm => resouce 
				=> cleanup <name_of_resource_group> 
				=> start <name_of_resource_group> 

	xxd   #see if a disk is EVMS (clustered) or not

################################################################################
#HAE pacemaker
    MYCMD3="/usr/sbin/crm_mon"
    MYCMD3OPTS="-1"

################################################################################
#VCS
    MYCMD4="/opt/VRTSvcs/bin/hastatus"
    MYCMD4OPTS="-sum"

#Run the command
if [[ -e $MYCMD1 ]]
then
        MYCMD1="$MYCMD1 $MYCMD1OPTS"
fi

################################################################################