#!/bin/sh 
#
# ASTIUM BIG BROTHER REPAIR SCRIPT
#

#
# call as: update_status "disk" "message"
#
BBLOG=/var/log/astium-bb.log
maillog() {
        DATE="`date`"
        echo "$DATE $*" | mail -s "astium `cat /etc/astium-serial`: $*" beheerq@neonova.nl
}
log() {
	DATE="`date` serial: `cat /etc/astium-serial`"
	echo "$DATE $*" >> $BBLOG
	maillog $1
}

#
# Collect information.
#

LOAD=`cat /proc/loadavg | cut -d' ' -f1 | cut -d. -f1`
if [ $LOAD -gt 4 ]
then
  log "load ($LOAD) too high to check services - skipping"
  exit 0
fi

ps ax |grep -v -i login > /tmp/bb.$$ 

PROCS="postmaster xinetd ntpd httpd sshd syslogd klogd "

CHECKPOSTFIX=`/sbin/chkconfig --list postfix 2>/dev/null | fgrep on`
if [ -n "$CHECKILLPOSTFIX" ]
then
  CHECKPOSTFIX=y
  PROCS="$PROCS postfix"
fi

CHECKNAMED=`/sbin/chkconfig --list named 2>/dev/null | fgrep on`
if [ -n "$CHECKNAMED" ]
then
  CHECKNAMED=y
  PROCS="$PROCS named"
fi

CHECKDHCPD=`/sbin/chkconfig --list dhcpd 2>/dev/null | fgrep on`
if [ -n "$CHECKDHCPD" ]
then
  CHECKDHCPD=y
  PROCS="$PROCS dhcpd"
fi

CHECKLDAP=`/sbin/chkconfig --list ldap 2>/dev/null | fgrep on`
if [ -n "$CHECKLDAP" ]
then
  CHECKLDAP=y
  PROCS="$PROCS slapd"
fi

CHECKPBX=`/sbin/chkconfig --list asterisk 2>/dev/null | fgrep on`
if [ -n "$CHECKPBX" ]
then
  CHECKPBX=y
  PROCS="$PROCS asterisk"
fi

CHECKNTPD=`/sbin/chkconfig --list ntpd 2>/dev/null | fgrep on`
if [ -n "$CHECKNTPD" ]
then
  CHECKNTPD=y
  PROCS="$PROCS ntpd"
fi

ASTCFGD=`/sbin/chkconfig --list astcfgd 2>/dev/null | fgrep on`
if [ -n "$ASTCFGD" ]
then
  ASTCFGD=y
  PROCS="astcfgd $PROCS"
fi

CHECKIAXMODEM=`/sbin/chkconfig --list iaxmodem 2>/dev/null | fgrep on`
if [ -n "$CHECKIAXMODEM" ]
then
  CHECKIAXMODEM=y
fi

CHECKHYLAFAX=`/sbin/chkconfig --list hylafax 2>/dev/null | fgrep on`
if [ -n "$CHECKHYLAFAX" ]
then
  CHECKHYLAFAX=y
  PROCS="$PROCS faxq hfaxd faxgetty"
fi

CHECKASTIUMD=`/sbin/chkconfig --list astiumd 2>/dev/null | fgrep on`
if [ -n "$CHECKASTIUMD" ]
then
  CHECKASTIUMD=y
  PROCS="$PROCS astiumd"
fi

CHECKNETWORK=`/sbin/chkconfig --list network 2>/dev/null | fgrep on`
if [ -n "$CHECKNETWORK" ]
then
  CHECKNETWORK=y
fi

CHECKNRPE=`/sbin/chkconfig --list nrpe 2>/dev/null | fgrep on`
if [ -n "$CHECKNRPE" ]
then
  CHECKNRPE=y
  PROCS="nrpe $PROCS"
fi

#
#=====================================================================
# PROCESSES THAT MUST EXIST
#=====================================================================
#
for proc in $PROCS
do
	fgrep "$proc" /tmp/bb.$$ | fgrep -v fgrep > /dev/null 2>&1
        if test "$?" != "0"
        then
		case $proc
		in
			postmaster)	/etc/rc.d/init.d/postgresql stop
					/etc/rc.d/init.d/postgresql start
					log "postgresql down, restarting !"
					;;
			slapd) 		/etc/rc.d/init.d/ldap stop
					/etc/rc.d/init.d/ldap start
					log "slapd down, restarting !"
					;;
			klogd) 		/etc/rc.d/init.d/syslog stop
					/etc/rc.d/init.d/syslog start
					log "syslog down, restarting !"
					;;
			syslogd)	/etc/rc.d/init.d/syslog stop
					/etc/rc.d/init.d/syslog start
					log "syslog down, restarting !"
					;;
			faxq|hfaxd|faxgetty)	/etc/rc.d/init.d/hylafax stop
					/etc/rc.d/init.d/hylafax start
					log "$proc down, restarting !"
					;;
			asterisk)       killall -9 mpg123
					/etc/rc.d/init.d/asterisk restart
					log "asterisk down, restarting !"
					maillog "asterisk down, restarting !"
                                        ;;
			*)		/etc/rc.d/init.d/$proc stop
					/etc/rc.d/init.d/$proc start
					log "$proc down, restarting !"
					;;
		esac	
	fi

done

#
#=====================================================================
# Reload asterisk moh if there are to many mpg123 processes.
#=====================================================================
#
MPG123COUNT=`ps ax|grep mpg123|grep -v grep|wc -l`
MPG123ALLOW=`cat /etc/asterisk/musiconhold.conf /etc/asterisk/company000*/musiconhold.conf | grep mode | wc -l`
MPG123ALLOW=$(($MPG123ALLOW*2))
if [ $MPG123COUNT -gt $MPG123ALLOW ]
then
        killall -9 mpg123
        sleep 5
        killall -9 mpg123
        /usr/sbin/asterisk -rx "moh reload"
        log "Too many mpg123 processes, killed, and reloaded asterisk moh!"
fi

#
#=====================================================================
# CHECK WORKINGS OF httpd
#=====================================================================
#
OUTPUT=`nbs-bbnet http://localhost`
if test "$?" != "0"
then
	/etc/rc.d/init.d/httpd stop
	/etc/rc.d/init.d/httpd start
	log "httpd down, restarting !"
fi

#
#=====================================================================
# CHECK WORKINGS OF asterisk
#=====================================================================
if [ "$CHECKPBX" = "y" ]
then
	ASTVERSION=`/usr/bin/astium-bb-asterisk`
	TEST=`echo $ASTVERSION|grep running|wc -l`
	if [ $TEST -eq 0 ]
	then
        	echo "Asterisk is not Running"
        	killall -9 safe_asterisk
        	killall -9 asterisk
        	killall -9 mpg123
		service asterisk restart

		log "asterisk down, restarting !"
		maillog "asterisk down, restarting !"
	fi
fi

#
#=====================================================================
# CHECK WORKINGS OF ntpd
#=====================================================================
#
if [ "$CHECKNTPD" = "y" ]
then
        COUNT=`ps ax|grep ntpd|wc -l`
        if [ $COUNT -lt 2 ]
        then
                /etc/rc.d/init.d/ntpd stop
                /etc/rc.d/init.d/ntpd start
                log "ntpd down, restarting !"
        fi
fi

#
#=====================================================================
# CHECK WORKINGS OF POSTFIX
#=====================================================================
#
OUTPUT=`nbs-bbnet localhost:25 "helo localhost"`
if test "$?" != "0"
then
        /etc/rc.d/init.d/postfix restart
	log "postfix down, restarting !"
fi

#
#=====================================================================
# CHECK WORKINGS OF SSHD
#=====================================================================
#
OUTPUT=`nbs-bbnet localhost:22`
if test "$?" != "0"
then
        /etc/rc.d/init.d/sshd restart
	log "sshd down, restarting !"
fi

#
#=====================================================================
# CHECK WORKINGS OF astiumd
#=====================================================================
# 
if [ "$CHECKASTIUMD" = "y" ]
then
        OUTPUT=`nbs-bbnet localhost:5656`
        if test "$?" != "0"
        then
                ASTPID=`ps aux | grep /sbin/astiumd | grep -v grep | awk '{ print $2 }'`
                date >> /var/log/astiumd.log
                gdb --batch -x 'bt full' -p $ASTPID >> /var/log/astiumd.log
                /etc/rc.d/init.d/astiumd restart
                log "astiumd down, restarting !"
        fi
fi

#
#=====================================================================
# CHECK WORKINGS OF iaxmodem
#=====================================================================
#
if [ "$CHECKIAXMODEM" = "y" ]
then
        COUNT=`ps ax|grep iaxmodem|wc -l`
        if [ $COUNT -lt 2 ]
        then
                /etc/rc.d/init.d/iaxmodem stop
                /etc/rc.d/init.d/iaxmodem start
                log "iaxmodem down, restarting !"
        fi
fi

#
#=====================================================================
# CHECK IF NETWORK IS UP (i.e. if we have a default gateway)
# NOTE: This does not work on BGP edge routers!!
#=====================================================================
#
if [ "$CHECKNETWORK" = "y" ]
then
        COUNT=`route -n | grep ^0.0.0.0 | wc -l`
        if [ $COUNT -eq 0 ]
        then
                /etc/rc.d/init.d/network restart
                log "network down, restarting !"
        fi
fi

# clean up tempfile
rm -f /tmp/bb.$$

