#!/bin/ksh
#
# Tivoli plugin to verify the BoKS database replication process
#
# Written by Thomas Sluyter (nagiosATkilalaDOTnl)
# By request of Unixerius, the Netherlands
# Last Modified: 16-03-2010
#
# Usage:
# ./check_boks_replication [-l LAG] [-h HOST] [-p] [-n] [-d -o FILE]
# -l LAG	Maximum amount of updates for a replica table to be behind on.
#			Typically this should not be over 50. Default is 30.
# -h HOST	Hostname of individual replica to verify.
# -x EXCLUDE	Hostname of replica to exclude.
# -p		No ping. Use if firewalls block ping.
# -n		Dry-run mode. Will only return an OK status.
# -d		Debug mode. Use with dry-run mode to test Tivoli.
# -o FILE	Output file for debugging logs. Required when -d is used.
# 
# Example: ./check_boks_sequence -l 20 -d -o /tmp/foobar
#
# Multiple -h and -x parameters are allowed.
#
# Description:
#   In a BoKS infrastructure the master server automatically distributes
# database updates to its replicas. BoKS provides the admin with a number
# of ways to verify the proper functioning of these replicas, but none
# of these is easily hooked into monitoring software.
#   This script makes use of the following methods to verify infra sanity.
# * boksdiag list, to verify if replicas are reachable.
# * boksdiag sequence, to verify if a replica's database is up to date.
#
# Installation:
#   This script must be installed in $BOKS_lib, on the master server and 
# ALL replica servers. It's needed on the replica host to do local se-
# quence checking. 
#
# Limitations:
# * This script should be run on the BoKS master server. From there it will
#   send commands to the replica servers to retrieve status information.
#   Running it on a replica server will only check the local sequence info.
#
# Setup:
#   Before using this monitoring script, please scroll down to the section
# labeled "SETTING THINGS UP". You'll need to make sure that a number of
# settings are correct for your BoKS installations, especially when it 
# comes to differences between OS platforms.
#
# Output:
# This script is meant to be called as a Tivoli numeric script.
# Hence both the output and the exit code are a single digit. Please
# configure your numeric script calls accordingly:
# 0     OK
# 1     WARNING
# 2     SEVERE
# 3     CRITICAL
#
# Other notes:
# *  Use the debugging option to find out the cause of your problems if this
#    script ever misbehaves. Use it with the -n dry run option if you like want
#    to test the script from within Tivoli, as opposed to on the command line.
#


### SETTING THINGS UP ###

PROGNAME=$(basename $0)
PATH="/usr/bin:/usr/sbin:/bin:/sbin:$PATH"
TEMPDIR="/tmp/$PROGNAME.temp"; mkdir $TEMPDIR > /dev/null 2>&1; rm $TEMPDIR/* >/dev/null 2>&1

MASTERSEQUENCE="$TEMPDIR/$(hostname).sequence"	# Tempfile that contains master status
MASTERDATABASE="$TEMPDIR/$(hostname).database"	# Tempfile that contains master status
REPLICASEQUENCE="SequenceStatus"		# Tempfile in $BOKS_tmp that contains sequence numbers
REPLICADATABASE="DatabaseStatus"		# Tempfile in $BOKS_tmp that contains database line counts
HOLDOFF="10"					# Seconds to wait before retrieving replica status

LAG="30"		# Default if -l parameter is not passed.
HOSTNAMES=""		# Default if -h parameter is not passed.
EXCLUDE=""              # Default if -x parameter is not passed.
NOPING="0"		# Default if -p parameter is not passed.
DEBUG="0"		# Default if -d parameter is not passed.
DEBUGFILE=""		# Default if -o parameter is not passed.
DRYRUN="0" 		# Default if -n parameter is not passed.

STATE_OK="0"            # Exit code when everything's OK
STATE_WA="1"            # Exit code when there might be a problem
STATE_SE="2"            # Exit code when there's a problem
STATE_CR="3"            # Exit code when there's a problem


# Setting up platform specific options

case $(uname) in
	AIX)    BOKSSBIN="/opt/boksm/sbin";
		BOKSBIN="/opt/boksm/bin";
		BOKSLIB="/opt/boksm/lib";
		BOKSTMP="/var/opt/boksm/tmp";
		BOKSETC="/etc/opt/boksm";
		AWK="/usr/bin/awk";
		PING="/bin/ping -c 1";
		PINGPARAM="";;
	Linux) BOKSSBIN="/opt/boksm/sbin";
		BOKSBIN="/opt/boksm/bin";
		BOKSLIB="/opt/boksm/lib";
		BOKSTMP="/var/opt/boksm/tmp";
		BOKSETC="/etc/opt/boksm";
		AWK="/bin/awk";
		PING="/bin/ping -c 1";
		PINGPARAM="";;
	SunOS)  BOKSSBIN="/opt/boksm/sbin";
		BOKSBIN="/opt/boksm/bin";
		BOKSLIB="/opt/boksm/lib";
		BOKSETC="/etc/opt/boksm";
		BOKSTMP="/var/opt/boksm/tmp";
		AWK="/usr/xpg4/bin/awk";
		PING="/usr/sbin/ping";
		PINGPARAM="1";;
	*)      [[ $DEBUG -gt 0 ]] && echo "OS not supported." >> $DEBUGFILE;
		echo "\nERROR: OS not supported."; exit 1 ;;
esac


# Quick check to see if you're running as root.

if [[ ( -z $(id | $AWK '{print $1}' | grep root) ) ]]
then
        echo "ERROR: Script must be run as root user.\n"
        exit 1
fi


### REQUISITE COMMAND LINE STUFF ###

function print_usage 
{
	echo "\nUsage: check_boks_sequence [-l LAG] [-h HOST] [-p] [-n] [-d -o FILE]"
	echo "\n-l LAG\t\tMaximum amount of updates for a replica table to be behind on."
	echo "\t\tTypically this should not be over 50. Default is 30."
	echo "-h HOST\t\tHostname of individual replica to verify."
	echo "-x EXCLUDE\tHostname of replica server to exclude."
	echo "-d\t\tDebug mode."
	echo "-o FILE\t\tOutput file for debugging logs. Required when -d is used."
	echo "-p\t\tDisables use of ping for connection testing, in case of firewalls."
	echo "-n\t\tPerforms a dry run, doing all the checks but returning an OK."
	echo "\nExamples:"
	echo "$PROGNAME -h HOST1 -h HOST2"
	echo "$PROGNAME -x HOST3 -n -d -o /tmp/debug.log"
	echo "\nMultiple -h and -x parameters are allowed."
	echo "\nThis script must be run as root on the BoKS master server\n"

	exit 1
}


while [[ $# -gt 0 ]]
do
    case "$1" in
	-l) if [[ (-z $(echo $2 | grep ^-)) && ($(echo $2) != "") ]]
	    then
		LAG="$2"; shift
	    else
		echo "\nERROR: Amount of lag incorrectly specified. Use -l.\n\n"
		print_usage
	    fi;;
	-h) if [[ (-z $(echo $2 | grep ^-)) && ($(echo $2) != "") ]]
	    then
		HOSTNAMES="$HOSTNAMES $2"; shift
	    else
		echo "\nERROR: Hostname incorrectly specified. Use -h.\n\n"
		print_usage
	    fi;;
	-x) if [[ (-z $(echo $2 | grep ^-)) && ($(echo $2) != "") ]]
	    then
		EXCLUDE="$EXCLUDE $2"; shift
	    else
		echo "\nERROR: Hostname to exclude incorrectly specified. Use -x.\n\n"
		print_usage
	    fi;;
	-o) if [[ (-z $(echo $2 | grep ^-)) && ($(echo $2) != "") ]]
	    then
		DEBUGFILE="$2"; shift
	    else
		echo "\nERROR: Debug log output file not specified. Use -o.\n\n"
		print_usage
	    fi;;
	-p) NOPING="1";;
	-d) DEBUG="1";;
	-n) DRYRUN="1";;
	-*) print_usage;;
	*)  print_usage;;	
    esac
    shift
done


# Setting up the debug log
if [[ $DEBUG -gt 0 ]]
then
	if [[ -d $DEBUGFILE ]] 
	then
		echo "\nERROR: Debug log output file not specified. Use -o.\n\n"
		print_usage
	else
		echo > $DEBUGFILE
	fi
 
	echo > $DEBUGFILE
	echo "=== SETUP ===" >> $DEBUGFILE
	echo "Script name is $PROGNAME" >> $DEBUGFILE
	echo "Script version is $VERSION" >> $DEBUGFILE
	echo "OS is $(uname -a)" >> $DEBUGFILE
	echo "Hostname is $(hostname)" >> $DEBUGFILE
	echo "BOKSSBIN is $BOKSSBIN" >> $DEBUGFILE
	echo "BOKSBIN is $BOKSBIN" >> $DEBUGFILE
	echo "BOKSLIB is $BOKSLIB" >> $DEBUGFILE
	echo "BOKSETC is $BOKSETC" >> $DEBUGFILE
	echo "" >> $DEBUGFILE
	echo "Parameters passed to script are:" >> $DEBUGFILE
	echo "LAG = $LAG" >> $DEBUGFILE
	echo "DRYRUN = $DRYRUN" >> $DEBUGFILE 
	echo "HOSTNAMES = $HOSTNAMES" >> $DEBUGFILE
	echo "EXCLUDE = $EXCLUDE" >> $DEBUGFILE
	[[ -f $EXCLUDE ]] && cat $EXCLUDE >> $DEBUGFILE
	echo "\nDEBUG = $DEBUG" >> $DEBUGFILE
	echo "DEBUGFILE = $DEBUGFILE" >> $DEBUGFILE
	echo "" >> $DEBUGFILE
fi


### DEFINING SUBROUTINES ###

function ExitScript
{
        EXIT="$1"

        rm -rf $TEMPDIR
        echo $EXIT
        exit $EXIT
}

function DetermineType
{
	# We need to ascertain whether we're running on the master,
	# a replica, or a client.

	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING DetermineType ===\n" >> $DEBUGFILE

	TYPE=$(grep ^BOKSINIT $BOKSETC/ENV | $AWK -F= '{print $2}')
	[[ $DEBUG -gt 0 ]] && echo "TYPE = $TYPE\n" >> $DEBUGFILE
}


function GenerateHostList
{
	# There are multiple ways to generate a list of master and replica servers for your
	# domain. One of them is to check the $BOKSETC/bcastaddr file, but often those files
	# get neglected. Two other ways are:
	# 1. Check the output of $BOKSLIB/boksdiag list. I'm using this one.
	# 2. Check table 6 of the BoKS database using $BOKSLIB/dumpbase for hosts with 
	#    TYPE="261".

	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING GenerateHostList ===\n" >> $DEBUGFILE

	if [[ -z $HOSTNAMES ]]
	then
       	[[ $DEBUG -gt 0 ]] && echo "No hostname passed. Going for all hosts." >> $DEBUGFILE
		HOSTNAMES=$($BOKSLIB/boksdiag list | grep -v Since | grep -v status | $AWK '{print $1}')
	else
       	[[ $DEBUG -gt 0 ]] && echo "Hostname(s) got passed from command line." >> $DEBUGFILE
	fi	

	if [[ -z $HOSTNAMES ]] 
	then
		[[ $DEBUG -gt 0 ]] && echo "Zero replica servers online.">> $DEBUGFILE
		ExitScript $STATE_SE
	fi
	
	NUMHOSTS=$(echo $HOSTNAMES | wc -w | $AWK '{print $1}')
	[[ $DEBUG -gt 0 ]] && echo "Checking $NUMHOSTS replicas." >> $DEBUGFILE
	[[ $DEBUG -gt 0 ]] && echo "$HOSTNAMES" >> $DEBUGFILE
}


function CheckBoksdiag
{
	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING CheckBoksdiag ===\n" >> $DEBUGFILE

	[[ $DEBUG -gt 0 ]] && $BOKSLIB/boksdiag list | grep $HOST >> $DEBUGFILE
	
	if [[ -z $($BOKSLIB/boksdiag list | grep $HOST | grep "ok") ]] 
	then
		[[ $DEBUG -gt 0 ]] && echo "Replica status for $HOST is NOK.">> $DEBUGFILE
		ExitScript $STATE_SE
	fi
}


function TestConnection
{
	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING TestConnection ===\n" >> $DEBUGFILE

	if [[ -z $($BOKSSBIN/hostadm -l -h $HOST | grep UNIXBOKSHOST | grep Replica) ]]
	then
		[[ $DEBUG -gt 0 ]] && echo "ERROR: $HOST is not a BoKS replica according to the database." >> $DEBUGFILE
		continue
	fi

	if [[ $NOPING -gt 0 ]]
	then
		PINGSTATUS="0"
	else
		IP=$($BOKSSBIN/hostadm -l -h $HOST | grep ^IP-address | $AWK '{print $2}')
		$PING $IP $PINGPARAM >/dev/null 2>&1
		PINGSTATUS=$?
	fi

	$BOKSLIB/cadm -l -f bcastaddr -h $HOST >/dev/null 2>&1
	CADMSTATUS=$?

	if [[ ($PINGSTATUS -gt 0) && ($CADMSTATUS -gt 0) ]]
	then
		[[ $DEBUG -gt 0 ]] && echo "ERROR: Host $HOST is unreachable." >> $DEBUGFILE

		ExitScript $STATE_SE
	fi
}


function ReplicaLocalCheck
{
	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING ReplicaLocalCheck ===\n" >> $DEBUGFILE
	
	$BOKSLIB/boksdiag sequence > $BOKSTMP/$REPLICASEQUENCE

        NUM=0; rm $BOKSTMP/$REPLICADATABASE
        let MAXNUM=$(boksdiag sequence | grep ^T | tail -1 | cut -c 2,3)+1
	while [[ $NUM -lt $MAXNUM ]]
	do
	    echo "$NUM:$(dumpbase -t$NUM | wc -l | $AWK '{print $1}')" >> $BOKSTMP/$REPLICADATABASE
	    let NUM=$NUM+1
	done
}


function ExecuteReplicaScript
{
	# Executing the check script on the remote replica
	# The ".nowait" tagged onto the end of $REPLICASCRIPT is courtesy of the cadm
	# man-page. It prevents the boks_clntd from waiting for the command to finish,
	# which could tie up things like password updates.

	if [[ $DEBUG -gt 0 ]] 
	then
		echo "\n=== STARTING ExecuteReplicaScript ===\n" >> $DEBUGFILE
		echo "$BOKSLIB/cadm -s $PROGNAME.nowait -h $HOST" >> $DEBUGFILE
	fi

	$BOKSLIB/cadm -s $PROGNAME.nowait -h $HOST 2>/dev/null
	
	if [[ $? -gt 0 ]] 
	then
		[[ $DEBUG -gt 0 ]] && echo "Execution of remote script on $HOST failed." >> $DEBUGFILE

		ExitScript $STATE_SE
	fi
}


function ReadReplicaSequence
{
	# Retrieving the status file from the remote replica.
	# The "\$BOKS_tmp" slash is used to let the replica use the actual
	# $BOKS_tmp variable on its side. 
	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING ReadReplicaSequence ===\n" >> $DEBUGFILE
	
	REPLICASEQSTATUS="$TEMPDIR/$HOST.sequence"
	REPLICADATSTATUS="$TEMPDIR/$HOST.database"
	
	[[ $DEBUG -gt 0 ]] && echo "$BOKSLIB/cadm -l -f \$BOKS_tmp/$REPLICASEQUENCE -h $HOST" >> $DEBUGFILE
	$BOKSLIB/cadm -l -f \$BOKS_tmp/$REPLICASEQUENCE -h $HOST > $REPLICASEQSTATUS 2>/dev/null

	[[ $DEBUG -gt 0 ]] && echo "$BOKSLIB/cadm -l -f \$BOKS_tmp/$REPLICADATABASE -h $HOST" >> $DEBUGFILE
	$BOKSLIB/cadm -l -f \$BOKS_tmp/$REPLICADATABASE -h $HOST > $REPLICADATSTATUS 2>/dev/null
		
	if [[ $? -gt 0 ]] 
	then
		[[ $DEBUG -gt 0 ]] && echo "Retrieval of output file from $HOST failed." >> $DEBUGFILE

                ExitScript $STATE_SE
	fi
}


function GetMasterStatus
{
	# Reading the master's status list.
	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING GetMasterStatus ===\n" >> $DEBUGFILE
	
	
	[[ $DEBUG -gt 0 ]] && echo "$BOKSLIB/boksdiag sequence > $MASTERSEQUENCE 2>/dev/null" >> $DEBUGFILE
	$BOKSLIB/boksdiag sequence > $MASTERSEQUENCE 2>/dev/null
	
	if [[ $? -gt 0 ]] 
	then
		[[ $DEBUG -gt 0 ]] && echo "Reading of master sequence on $(hostname) failed." >> $DEBUGFILE

                ExitScript $STATE_SE
	fi

	[[ $DEBUG -gt 0 ]] && echo "echo \"\$NUM:\$(dumpbase -t\$NUM | wc -l)\" >> \$MASTERDATABASE" >> $DEBUGFILE

        NUM=0
        let MAXNUM=$(boksdiag sequence | grep ^T | tail -1 | cut -c 2,3)+1

        while [[ $NUM -lt $MAXNUM ]]
        do
            echo "$NUM:$(dumpbase -t$NUM | wc -l | $AWK '{print $1}')" >> $MASTERDATABASE
            let NUM=$NUM+1
        done
}


function CompareSequence
{
	# Now comes the hard part: comparing the sequence numbers between the master and
	# the replica. Sequence numbers are not guaranteed to be identical, but a slight
	# difference is not a problem. Guaranteed error conditions are:
	# * If a replica server is ahead of the master server.
	# * If the master server is more than $LAG updates ahead of a replica.
	
	[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING CompareSequence ===\n" >> $DEBUGFILE

	cat $REPLICASEQSTATUS | $AWK '{print $1" "$4}' | while read TABLE REPSEQ
	do
		[[ $TABLE = "LOG" ]] && continue
		
		MASTSEQ=$(grep ^"$TABLE " $MASTERSEQUENCE | $AWK '{print $4}' | tr -d "(" | tr -d ")")
		REPSEQ=$(echo $REPSEQ | tr -d "(" | tr -d ")")
		
		[[ $DEBUG -gt 0 ]] && echo "Processing table $TABLE (M=$MASTSEQ R=$REPSEQ)." >> $DEBUGFILE

		[[ ( -z $REPSEQ ) || ( -z $MASTSEQ ) ]] && ExitScript $STATE_WA

		if [[ $REPSEQ -gt $MASTSEQ ]]
		then
			[[ $DEBUG -gt 0 ]] && echo "Replica $HOST is ahead of master on table $TABLE." >> $DEBUGFILE

	                ExitScript $STATE_WA
		fi
		
		let DIFF=$MASTSEQ-$REPSEQ
		
		if [[ ( $MASTSEQ -gt $REPSEQ ) && ( $DIFF -gt $LAG ) ]]
		then
			[[ $DEBUG -gt 0 ]] && echo "Table $TABLE on replica $HOST is lagging behind. Max is $LAG." >> $DEBUGFILE

	                ExitScript $STATE_WA
		fi
	done
}


function CompareDatabase
{
        # Another hard part: comparing the number of database entries between the master and
        # the replica. These should match the sequence numbers, but unfortunately sequence
	# numbers are not updated if a file gets accidentaly deleted. However, a slight
        # difference is not a problem. Guaranteed error conditions are:
	# * If a replica server has 0 lines in an otherwise full table.
	# * If a replica server is more than a handful of lines behind the master.
	# * If a replica server is ahead of the master server.

        if [[ $DEBUG -gt 0 ]] 
	then
		echo "\n=== STARTING CompareDatabase ===\n" >> $DEBUGFILE
        	echo "\nComparing $MASTERDATABASE with $REPLICADATSTATUS" >> $DEBUGFILE
        	ls -al $MASTERDATABASE >> $DEBUGFILE
        	ls -al $REPLICADATSTATUS >> $DEBUGFILE
	fi

        cat $MASTERDATABASE | $AWK -F: '{print $1" "$2}' | while read TABLE MASTLINES
        do
                [[ $TABLE = "LOG" ]] && continue

                REPLINES=$(grep ^"$TABLE:" $REPLICADATSTATUS | $AWK -F: '{print $2}')

                [[ $DEBUG -gt 0 ]] && echo "Processing table $TABLE (M=$MASTLINES R=$REPLINES)." >> $DEBUGFILE

                [[ ( -z $REPLINES ) || ( -z $MASTLINES ) ]] && ExitScript $STATE_WA

                if [[ $REPLINES -gt $MASTLINES ]]
                then
                        [[ $DEBUG -gt 0 ]] && echo "Replica $HOST is ahead of master on table $TABLE." >> $DEBUGFILE

                        ExitScript $STATE_WA
                fi

                let DIFF=$MASTLINES-$REPLINES

                if [[ ( $MASTLINES -gt $REPLINES ) && ( $DIFF -gt $LAG ) ]]
                then
                        [[ $DEBUG -gt 0 ]] && echo "Table $TABLE on replica $HOST is lagging behind. Max is $LAG." >> $DEBUGFILE

                        ExitScript $STATE_WA
                fi
        done
}


### FINALLY, THE MAIN ROUTINE ###

[[ $DEBUG -gt 0 ]] && echo "\n=== STARTING MAIN PHASE ===\n$(date)\n" >> $DEBUGFILE

COUNT="0"; FAIL="0"

DetermineType

case $TYPE in
	master) GenerateHostList;

		if [[ $(echo $EXCLUDE) != "" ]]
		then
			NUMEXCLUDE=$(echo $EXCLUDE | wc -w | $AWK '{print $1}')

			if [[ $DEBUG -gt 0 ]] 
			then
			    echo "Excluding $NUMEXCLUDE servers." >> $DEBUGFILE
			    echo "$EXCLUDE" >> $DEBUGFILE
			fi
		fi

		for HOST in $(echo $HOSTNAMES)
		do
			[[ $DEBUG -gt 0 ]] && echo "\n=== Starting MASTER loop for $HOST ===\n" >> $DEBUGFILE

        		if [[ ! -z $(echo $EXCLUDE | grep $HOST) ]]
        		then
			[[ $DEBUG -gt 0 ]] && echo "WARNING: Host $HOST is on exclude list. Skipping." >> $DEBUGFILE
			continue
        		fi

        		TestConnection

        		CheckBoksdiag 

        		ExecuteReplicaScript; sleep $HOLDOFF

		        ReadReplicaSequence

		        GetMasterStatus

		        CompareSequence
		        CompareDatabase

		        let COUNT=$COUNT+1
		done

		[[ $DEBUG -gt 0 ]] && echo "\n\n=== SCRIPT FINISHED ===\n\n" >> $DEBUGFILE;
		[[ $DEBUG -gt 0 ]] && echo "Ran for $COUNT servers ($NUMHOSTS total, $NUMEXCLUDE excluded).\n" >> $DEBUGFILE;
		[[ $DEBUG -gt 0 ]] && date >> $DEBUGFILE;;

	replica) ReplicaLocalCheck;; 

	*) 	[[ $DEBUG -gt 0 ]] && echo "ERROR: $(hostname) is not a master or replica." >> $DEBUGFILE;
		ExitScript $STATE_WA;;
esac


ExitScript $STATE_OK

