#!/bin/bash
# check if all backups succeeded. Run every night on Astium 
# backup server only, after all backups have been executed.
# Also creates weekly backup archive in /home/<serial>/archive
# This means we currently save backups for a week, not longer.

# Include some functions.
. /usr/local/astium/sbin/astium-backup-archive-lib.sh

# Create temp working directory.
create_temp_dir

# Create astium list.
create_astium_list

# Set logfile
logfile="$TEMPDIR/backup-check.log"
xmlfile="$TEMPDIR/backup-check.xml"

# Reset global astium backup error to 0
backup_error=0

# Call pre backuparchive script.
if [ -x /usr/local/astium/sbin/astium-backup-pre.sh ]
then
	. /usr/local/astium/sbin/astium-backup-pre.sh
fi

# Check if the backup server config file exists.
backup_server_conf="/usr/local/astium/etc/astium-backup-server.cfg"
if [ -f $backup_server_conf ]
then
        # Check if we have a email_adress entry.
        test=`grep ^email_adress= $backup_server_conf`
        if [ -n "$test" ]
        then
                EMAIL=`grep ^email_adress= $backup_server_conf|sed -e "s/^email_adress.*=\"//" -e "s/\".*$//g"`
        else
                EMAIL="beheerq@neonova.nl"
        fi

        # Check if we have to test the md5sum.
        test=`grep ^check_md5sum= $backup_server_conf`
        if [ -n "$test" ]
        then
                CHECK_MD5SUM=`grep ^check_md5sum= $backup_server_conf|sed -e "s/^check_md5sum.*=\"//" -e "s/\".*$//g"`
        else
                CHECK_MD5SUM="true"
        fi
else
        EMAIL="beheerq@neonova.nl"
	CHECK_MD5SUM="true"
fi

#
# Reset astium backup error for each reseller to 0
#
for reseller_id in `cat $ASTIUM_RESELLER_LIST |awk -F, '{ print $1 }'|sort|uniq`
do
	# reseller_backup_error file.
	reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"

	# Reset reseller_backup_error.
	echo "0" > $reseller_backup_error
done 

#
# Process each astium.
#
for serial in `cat $ASTIUMLIST_LOCAL`
do
	# Get reseller id.
	reseller_id=`grep ^$serial $ASTIUM_LIST |awk -F, '{ print $4 }'`

	# Set logfile for the reseller.
	logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"

	# Data string.
	date=`date`

	# Reset some values.
	astium_backup_error=0

	# Set backup log for a specific astium.
	backup_log="/home/$serial/backup.log"

	# Check if a backup directory exists.
	if [ ! -d /home/$serial ]
	then
		# Only create the home directory when the user exists in the passwd file.
        	test=`grep "^$serial:" /etc/passwd`
        	if [ -n $test ]
        	then
			# home directory does not exist, so we create it.
			mkdir -p /home/$serial
			chmod 700 /home/$serial
			chown $serial:$serial /home/$serial

			echo "" >> $backup_log
			astium_backup_check_log "Fatal error, could not find homedir for astium."
			astium_backup_check_log "Created backup directory, so the backup might work the next time."
			astium_backup_error=1
		fi

	else 	 
		echo "" >> $backup_log
		astium_backup_check_log "Starting backup check"
		astium_backup_error=0
	fi

	# If we could not find the backup directories then backup check is not needed.
	if [ $astium_backup_error = 0 ]
	then

		#
		# Create modules list.
		#
		modules="data databases logfiles sysinfo system"
	
		# If check_md5sum is enabled add md5sum to the modules list.	
		if [ $CHECK_MD5SUM = "true" ]
		then
			modules="$modules md5sum"
		fi	

		#
		# Here we check the exit files for each backup module.
		#
		for module in data $modules
		do
			# Reset module error.
			module_backup_error=0

			# Setting exit filename for the combination serial and module.	
			FILENAME="/home/$serial/upload/$module/.exit"		
			MODULE="/home/$serial/upload/$module/"

			# First check if there is a exit file.	
			if [ ! -f $FILENAME ]
			then
				module_backup_error=1
				astium_backup_check_log "module: $module: No .exit file found."
			else 
				# Check if the exit file is not empty.
				file_size=`ls $FILENAME -l|awk '{ print $5 }'`
				if [ $file_size -eq 0 ]
				then
					module_backup_error=1
					astium_backup_check_log "module: $module: empty .exit file found."
				fi
			
				# Check if the exit file is not older than 1 day.	
				if [ $module_backup_error = 0 ]
				then
					test1=`find $FILENAME -type f -mtime +1|grep -v archive`
					if [ -n "$test1" ]
					then
						module_backup_error=1
						astium_backup_check_log "module: $module: .exit file found older than 1 days."
					fi
				fi
			
				# Check if the exit file contains a valid number (0 or 1). 	
				if [ $module_backup_error = 0 ]
				then
					test=`cat $FILENAME|grep -v ^$ |sed s/:.*//g|grep -v "^[01]$"`
					if [ -n "$test" ]
					then
						module_backup_error=1
						astium_backup_check_log "module: $module: .exit file contains one or more invalid exits codes."
					fi
				fi
				
				# If exit value is equal to 1 then backup failed.
				if [ $module_backup_error = 0 ]
				then
					test=`cat $FILENAME |grep -v ^$ |sed s/:.*//g|grep "^[1]$"`
					if [ -n "$test" ]
					then
						module_backup_error=1
						astium_backup_check_log "module: $module: .exit gave backup error 1."
					fi
				fi
			fi

			# Check al modules, but not the data and logfiles module.
			if [ ! $module = "data" -a ! $module = "logfiles" ]
			then

				# check if module directory exists.
				if [ $module_backup_error -eq 0 ]
				then
					if [ ! -d $MODULE ]
					then
						module_backup_error=1
					fi	
				fi

				# check for empty directory.
				if [ $module_backup_error -eq 0 ]
				then
					count=`ls $MODULE|wc -l`
 		               		if [ $count -eq 0 ]
                			then
						module_backup_error=1
						astium_backup_check_log "module: $module: No backup files found."
                			fi
				fi

				# Check that al files are not empty.
        			if [ $module_backup_error -eq 0 ]
        			then
                			for file in `ls $MODULE`
                			do
                        			file_size=`ls $MODULE/$file -l|awk '{ print $5 }'`
                        			if [ $file_size -eq 0 ]
                        			then
							module_backup_error=1
							astium_backup_check_log "module: $module: Empty files found."
                        			fi
                			done
        			fi

				# check for files older than 1 day.
        			if [ $module_backup_error -eq 0 ]
        			then
					test=`find $MODULE -type f -mtime +1|grep -v archive`
                			if [ -n "$test" ]
                			then
						module_backup_error=1
						astium_backup_check_log "module: $module: Files found older than 1 days."
					fi
				fi
			fi

			# Now we log if the backup is OK! or NOT OK!	
			if [ $module_backup_error -eq 0 ]
			then
				astium_backup_check_log "module: $module: OK!"
			else 
				astium_backup_check_log "module: $module: FAILED!"
				astium_backup_error=1
			fi
		done
	fi

        # Check the astium backup_pwd md5sum from from cms with that from the backup.
	if [ "$CHECK_MD5SUM" = "true" ]
	then
        	if [ -f /home/$serial/upload/md5sum/backuppasswd-md5sum ]
		then
                	MD5SUM_CMS=`cat $ASTIUM_LIST|grep ^$serial |awk -F, '{ print $5 }'`
                	MD5SUM_LOC=`cat /home/$serial/upload/md5sum/backuppasswd-md5sum`
	
                	# If we dont have the correct password, give a backup error.
                	if [ "$MD5SUM_CMS" != "$MD5SUM_LOC" ]
                	then
				astium_backup_check_log "module: md5sum: FAILED! : Incorrect astium backup pwd in cms!"
				astium_backup_error=1
                	else 
				astium_backup_check_log "module: md5sum: OK!"
			fi
        	else 
			astium_backup_check_log "module: md5sum: FAILED! : Could not find the md5sum file!"
			astium_backup_error=1
		fi
	fi

	# Backup check is done for this astium.
	astium_backup_check_log "Finished backup check"

	# If backup ok, then create backup archive.
	if [ $astium_backup_error -eq 0 ]
	then
		day=`date +%A`
		upload="/home/$serial/upload"
		archive="/home/$serial/archive/daily"

		# Remove old tmp directory.		
		if [ -d $archive/$day.tmp ]
		then
			rm -rf $archive/$day.tmp
		fi

		# Move old directory to tmp directory.	
		if [ -d $archive/$day ]
		then
			mv $archive/$day $archive/$day.tmp
		fi

		# Copy uploaded files to archive.	
		for module in databases logfiles sysinfo system
		do
			mkdir -p $archive/$day/$module
			cp -af $upload/$module/* $archive/$day/$module
		done

                # Compress the logfile to safe disk space.
		if [ -d "$archive/$day/logfiles" ]
		then
                	gzip -f -9 -r $archive/$day/logfiles
		fi

		# Copy uploaded files to archive (using hardlinks, to safe disk space).	
		mkdir -p $archive/$day/data
		cp -alf $upload/data/* $archive/$day/data

		# Copy the backup_pwd md5sum to archive.	
		if [ -d $upload/md5sum ]
		then
			mkdir -p $archive/$day/md5sum
			cp -alf $upload/md5sum/* $archive/$day/md5sum
		fi

		# Set the correct owner for the backup.
		chown -R $serial:$serial $archive/$day

		# Remove tmp directory.	
		if [ -d $archive/$day.tmp ]
		then
			rm -rf $archive/$day.tmp
		fi

		# Add backup date (daily archive directory).
		date +%s > $archive/$day/.backup_date

                # Copy extra information.
                test=`ls -all $upload/|grep .backup_client_id`
                if [ -n "$test" ]
                then
                        cp $upload/.backup_client_id*  	$archive/$day 
                fi

                if [ -f $upload/.backup_client_name ]
                then
                        cp $upload/.backup_client_name 	$archive/$day 
                fi

		# Add backup status (daily archive directory).
		echo "OK!" > $archive/$day/.backup_status
	fi

	#
	# Add the backup status to the upload directory.
	#
	if [ $astium_backup_error -eq 0 ]
	then
		# Add backup status (upload directory).
		echo "OK!" > $upload/.backup_status

		# Add backup date upload directory.
		date +%s > $upload/.backup_date
	else 
		# Add backup status (upload directory).
		echo "NOT OK" > $upload/.backup_status
	fi

	#
	# Create weekly archive directory by means of rsync and using hardlinks to save space.
	#
	if [ `date +%A` = 'Sunday' ]
	then	
		# Calculate weekly backup number (1,2,3 or 4).
		week_nr=`date +%V`
		a=`expr $week_nr % 4`
		b=`expr $a + 1`
		week_nr=`printf "%02d" $b`
		archive="/home/$serial/archive/daily"

		weekly_archive="/home/$serial/archive/weekly/$week_nr"
	
		if [ ! -d $weekly_archive ]
		then
			mkdir -p $weekly_archive
		fi
	
		if [ -d /home/$serial/archive/daily ]
		then
			rsync -H -avz --hard-links --delete $archive/ $weekly_archive	
		fi
	fi

	# Get astium name.
	name=`cat $ASTIUM_LIST |grep ^$serial|awk -F, '{ print $3 }'`

	if [ $astium_backup_error -eq 1 ]
	then
		text="Backup Astium $serial : $name : NOT OK! (look in /home/$serial/backup.log for more information)"
		echo $text >> $logfile.not_ok

		text="Backup Astium $serial : $name : NOT OK!"
		echo $text >> $logfile_reseller.not_ok

		# Set reseller_backup_error.
		reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"
		echo "1" > $reseller_backup_error

		# Set astium backup_error.
		backup_error=1

		# Add backup status to xml file.
		echo "$serial:1" >> $xmlfile
	else
		text="Backup Astium $serial : $name : OK"
		echo $text >> $logfile.ok
		echo $text >> $logfile_reseller.ok

		# Add backup status to xml file.
		echo "$serial:0" >> $xmlfile
	fi	
	
done

#
# Create list of astiums where the backup is not activated.
#
for serial in `cat $ASTIUM_LIST|grep "[0-9]\{5\}"|grep -v false |grep -v disabled|awk -F, '{ print $1 }'|sort|uniq`
do
        test=`grep "^$serial:" /etc/passwd`
        if [ -z $test ]
        then
		# Get astium name.
		name=`cat $ASTIUM_LIST |grep ^$serial|awk -F, '{ print $3 }'`

                text="Backup Astium $serial : $name : Not Activated"
		echo $text >> $logfile.not_activated

		# Get reseller id.
		reseller_id=`grep ^$serial $ASTIUM_LIST |awk -F, '{ print $4 }'`

		# Set logfile for the reseller.
		logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"
		echo $text >> $logfile_reseller.not_activated
	
		# Set astium backup_error.
                backup_error=1

		# Set reseller_backup_error.
		reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"
		echo "1" > $reseller_backup_error
	fi
done

#
# Create list of astiums where the backup is disabled.
#
for serial in `cat $ASTIUM_LIST|grep "[0-9]\{5\}"|grep disabled |awk -F, '{ print $1 }'`
do
	# Get astium name.
	name=`cat $ASTIUM_LIST |grep ^$serial|awk -F, '{ print $3 }'`

	# Get reseller id.
	reseller_id=`grep ^$serial $ASTIUM_LIST |awk -F, '{ print $4 }'`

	# Set logfile for the reseller.
	logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"

	text="Backup Astium $serial : $name : Backup disabled"
	echo $text >> $logfile.disabled
	echo $text >> $logfile_reseller.disabled
done

#
# Here we send the email to beheer@neonova.nl
#

# Set Subject for astium backup mail.
if [ $backup_error -eq 1 ]
then
	subject="Astium backup report: Some backups failed! Please investigate and fix ASAP!" 
else 
	subject="Astium backup report: All backups are OK!" 
fi

#
# Get email adres for the backup report.
#

# Create Email message.
if [ -f $logfile.not_ok ]
then
	cat $logfile.not_ok		>> $logfile
	echo "" 			>> $logfile
fi

DAYOFWEEK=`date +%A`
if [ $DAYOFWEEK = "Monday" ]
then
	if [ -f $logfile.disabled ]
	then
		cat $logfile.disabled	>> $logfile
	fi
fi

if [ -f $logfile.not_activated ]
then
	cat $logfile.not_activated	>> $logfile
	echo "" 			>> $logfile
fi

if [ -f $logfile.ok ]
then
	cat $logfile.ok			>> $logfile
	echo "" 			>> $logfile
fi

# Send the backup report email.
cat $logfile|mail $EMAIL -s "$subject"

#
# Here we send the email to all resellers.
#
for reseller_id in `cat $ASTIUM_RESELLER_LIST |awk -F, '{ print $1 }'|sort|uniq`
do

	# Check if the reseller has a astium or sold a astium.
	test=`cat $ASTIUM_LIST|awk -F, '{ print  $4 }'|sort|uniq|grep $reseller_id`
	if [ -n "$test" ]
	then

		# reseller_backup_error file.
		reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"

		# Create empty reseller email.
		logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"
		> logfile_reseller

		# Get backup error.
		backup_error=`cat $reseller_backup_error`

		# Create reseller backup mail.
		reseller_backup_report="$TEMPDIR/reseller-backup-report-$reseller_id.txt"
		> $reseller_backup_report

		#
		# Adding Subject to the reseller email.	
		#
		if [ $backup_error -eq 1 ]
		then
			subject="Astium backup report: Some backups failed!"
			echo "Subject: $subject" >> $reseller_backup_report
		else
       		 	subject="Astium backup report: All backups are OK!"
			echo "Subject: $subject" >> $reseller_backup_report
		fi

		#
		# Adding email adressen to the reseller email.
		#
		for email_adres in `grep ^$reseller_id $ASTIUM_RESELLER_LIST|awk -F, '{ print $2 }'`
		do
			echo "To:$email_adres" 			>> $reseller_backup_report
		done
		echo "Bcc:perry@couprie.net"  			>> $reseller_backup_report

		# Add message header.
		echo "" 					>> $reseller_backup_report
		add_backup_mail_header 				$reseller_backup_report

		if [ -f $logfile_reseller.not_ok ]
		then
			echo "" 				>> $reseller_backup_report
			cat $logfile_reseller.not_ok		>> $reseller_backup_report
		fi

		if [ -f $logfile_reseller.not_activated ]
		then
		echo "" 					>> $reseller_backup_report
			cat $logfile_reseller.not_activated	>> $reseller_backup_report
		fi

		if [ -f $logfile_reseller.ok ]
		then
		echo "" 					>> $reseller_backup_report
			cat $logfile_reseller.ok		>> $reseller_backup_report
		fi

		if [ -f $logfile_reseller.disabled ]
		then
			echo "" 				>> $reseller_backup_report
			cat $logfile_reseller.disabled		>> $reseller_backup_report
		fi

		# Add mail footer.
		echo "" 					>> $reseller_backup_report
		add_backup_mail_footer 				$reseller_backup_report

		# Send the email we created to the reseller.
		cat $reseller_backup_report|/usr/sbin/sendmail -fbeheer@neonova.nl -F"NeoNova Beheer" -t
	fi
done 

# Create a backup list for each astium. with all the backup that are in the daily and weekly archive.
for serial in `ls /home/|grep "[0-9]\{5\}"`
do
	> /tmp/backup-$serial.txt
	for file in `find /home/$serial/archive/ -name .backup_date -type f`
	do
		epoch_time=`cat $file`
		backup_directory=`dirname $file|sed -e s#/home/$serial/archive/##g`
		echo "$epoch_time,$backup_directory" >> /tmp/backup-$serial.txt
	done

	# Move old list to .old, overwrite if needed.
	if [ -f /home/$serial/.backup_lists ]
	then
		mv -f /home/$serial/.backup_lists /home/$serial/.backup_lists.old	
	fi

	# Create a list of all backups.
	> /home/$serial/.backup_lists
	if [ -f /tmp/backup-$serial.txt ]
	then
		cat /tmp/backup-$serial.txt|sort -n > /home/$serial/.backup_lists
	fi
done

# Remove temp files left by the daily rsync processes.
for serial in `ls /home/|grep "[0-9]\{5\}"|grep -v ^n`
do
        if [ -d "/home/$serial/.tmp/" ]
        then
                rm -vf /home/$serial/.tmp/*
        fi
done

# Call post backup script.
if [ -x /usr/local/astium/sbin/astium-backup-post.sh ]
then
	. /usr/local/astium/sbin/astium-backup-post.sh
fi

# Remove temp working directory.
#remove_temp_dir
