#!/bin/bash
# check if all backups succeeded. Run every night on the 
# backup server only, after all backups have been executed.
# Also creates weekly backup archive in /home/<serial>/archive
# This means we currently save backups for a week, not longer.

# Backup start.
time_start=`date +%s`

# Renice script.
if [ -x /usr/bin/renice ]
then
	renice -20 -p $$
fi

# Do not be nice to the disks.
if [ -x /usr/bin/ionice ]
then
	ionice -n0 -c1 -p $$
fi

# Include some functions.
. /usr/local/neonova/sbin/neonova-backup-archive-lib.sh

# Try to get a lock file otherwise quit.
getlockfile-backup-archive

# Create temp working directory.
create_temp_dir

# Call pre backup script.
if [ -x /usr/local/neonova/sbin/neonova-backup-pre.sh ]
then
	. /usr/local/neonova/sbin/neonova-backup-pre.sh
fi

# Create server list.
create_neonova_list

# Set logfile
logfile="$TEMPDIR/backup-check.log"
xmlfile="$TEMPDIR/backup-check.xml"

## Reset global server backup error to 0
backup_error=0

# Get backup archive config options.
get_backup_archive_config

#
# Reset server backup error for each reseller to 0
#
for reseller_id in `cat $NEONOVA_RESELLER_LIST |awk -F, '{ print $1 }'|sort|uniq`
do
	# reseller_backup_error file.
	reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"

	# Reset reseller_backup_error.
	echo "0" > $reseller_backup_error
done 

#
# Process each server.
#
for serial in `cat $NEONOVALIST_LOCAL`
do
	# Get reseller id.
	reseller_id=`grep ^$serial $NEONOVA_LIST |awk -F, '{ print $4 }'`

	# Set logfile for the reseller.
	logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"

	# Data string.
	date=`date`

	# Reset some values.
	server_backup_error=0

	# Set backup log for a specific server.
	backup_log="/home/$serial/backup.log"

	#
	# Check if a backup directory exists.
	#
	if [ ! -d /home/$serial ]
	then
		# Only create the home directory when the user exists in the passwd file.
	       	test=`grep "^$serial:" /etc/passwd`
        	if [ -n $test ]
        	then
			# home directory does not exist, so we create it.
			mkdir -p /home/$serial
			chmod 700 /home/$serial
			chown $serial:$serial /home/$serial

			echo "" >> $backup_log
			neonova_backup_check_log "Fatal error, could not find homedir for server."
			neonova_backup_check_log "Created backup directory, so the backup might work the next time."
			server_backup_error=1
		fi
	else 	 
		echo "" >> $backup_log
		neonova_backup_check_log "Starting backup check"
		server_backup_error=0
	fi

	# remove backup_client_id files where the change time is longer dan 2 days ago.
	test=`ls /home/$serial/upload/ -a|grep .backup_client_id`
	if [ -n "$test" ]
	then
        	find /home/$serial/upload/.backup_client_id* -type f -cmin +1980|xargs rm -vf
	fi

	# Check if there are multiple backup clients using the same backup serial.
	if [ $server_backup_error = 0 ]
	then
		backup_clients_count=`ls /home/$serial/upload/.backup_client_id* 2>&1|grep .backup_client|wc -l`
		if [ $backup_clients_count -gt 1 ]
		then
			neonova_backup_check_log "Backup FAILED! Multiple clients using same serial number."
			server_backup_error=1
		fi
	fi

	#
	# If we could not find the backup directories then backup check is not needed.
	#
	if [ $server_backup_error = 0 ]
	then

		#
		# Create modules list.
		#
		modules="data databases logfiles sysinfo system"
	
		# If check_md5sum is enabled add md5sum to the modules list.	
		if [ $CHECK_MD5SUM = "true" ]
		then
			modules="$modules md5sum"
		fi	

		#
		# Here we check the exit files for each backup module.
		#
		for module in $modules
		do
			# Reset module error.
			module_backup_error=0
		
			# Reset modules status.
			module_status=""

			# Setting exit filename for the combination serial and module.	
			FILENAME="/home/$serial/upload/$module/.exit"		
			MODULE="/home/$serial/upload/$module/"

			# First check if there is a exit file.	
			if [ ! -f $FILENAME ]
			then
				module_backup_error=1
				neonova_backup_check_log "module: $module: No .exit file found."
			else 
				# Check if the exit file is not empty.
				file_size=`ls $FILENAME -l|awk '{ print $5 }'`
				if [ $file_size -eq 0 ]
				then
					module_backup_error=1
					neonova_backup_check_log "module: $module: empty .exit file found."
				fi
			
				# Check if the exit file is not older than 1 day.	
				if [ $module_backup_error = 0 ]
				then
					test1=`find $FILENAME -type f -mtime +1|grep -v archive`
					if [ -n "$test1" ]
					then
						module_backup_error=1
						neonova_backup_check_log "module: $module: .exit file found older than 1 days."
					fi
				fi

                                # Check if the exit file contains a valid number (0 or 1).      
                                if [ $module_backup_error = 0 ]
                                then
                                        test=`cat $FILENAME|grep -v ^$ |sed s/:.*//g|grep -v "^[01]$"`
                                        if [ -n "$test" ]
                                        then
                                                module_backup_error=1
                                                neonova_backup_check_log "module: $module: .exit file contains one or more invalid exits codes."
                                        fi
                                fi

                                # If exit value is equal to 1 then backup failed.
                                if [ $module_backup_error = 0 ]
                                then
                                        test=`cat $FILENAME |grep -v ^$ |sed s/:.*//g|grep "^[1]$"`
                                        if [ -n "$test"  ]
                                        then
                                                module_backup_error=1
                                                neonova_backup_check_log "module: $module: .exit gave backup error 1."
                                        fi
                                fi
                        fi

			# Check al modules, but not the data and logfiles module.
			if [ ! $module = "data" -a ! $module = "logfiles" ]
			then

				# Check if the module can be empty.
				module_status=`cat $FILENAME|grep -v ^$|grep '^[0-9]:'|grep empty|sed -e "s/^[0-9:]*//g"|awk -F: '{ print $1 }'|tail -1`
				if [ ! $module_status = "empty" ]
				then

					# check if module directory exists.
					if [ $module_backup_error -eq 0 ]
					then
						if [ ! -d $MODULE ]
						then
							module_backup_error=1
						fi	
					fi

					# check for empty directory.
					if [ $module_backup_error -eq 0 ]
					then
						count=`ls $MODULE|wc -l`
 			               		if [ $count -eq 0 ]
 		              			then
							module_backup_error=1
							neonova_backup_check_log "module: $module: No backup files found."
		               			fi
					fi

					# Check that al files are not empty.
        				if [ $module_backup_error -eq 0 ]
        				then
                				for file in `ls $MODULE`
                				do
                        				file_size=`ls $MODULE/$file -l|awk '{ print $5 }'`
                        				if [ $file_size -eq 0 ]
                        				then
								module_backup_error=1
								neonova_backup_check_log "module: $module: Empty files found."
                        				fi
                				done
        				fi

					# check for files older than 1 day.
	       				if [ $module_backup_error -eq 0 ]
	       				then
						test=`find $MODULE -type f -mtime +1|grep -v archive`
                				if [ -n "$test" ]
	           	    			then
							module_backup_error=1
							neonova_backup_check_log "module: $module: Files found older than 1 days."
						fi
					fi
				fi
			fi

			# Now we log if the backup is OK! or NOT OK!	
			if [ $module_backup_error -eq 0 ]
			then
				if [ ! -z "$module_status" -a "$module_status" = "empty" ]
				then
					neonova_backup_check_log "module: $module check skipped: OK!"
				else
					neonova_backup_check_log "module: $module: OK!"
				fi
			else 
				neonova_backup_check_log "module: $module: FAILED!"
				server_backup_error=1
			fi
		done
	fi

	#
        # Check the server backup_pwd md5sum from from cms with that from the backup.
	#
	if [ "$CHECK_MD5SUM" = "true" ]
	then
 	      	if [ -f /home/$serial/upload/md5sum/backuppasswd-md5sum ]
		then
                	MD5SUM_CMS=`cat $NEONOVA_LIST|grep ^$serial |awk -F, '{ print $5 }'`
                	MD5SUM_LOC=`cat /home/$serial/upload/md5sum/backuppasswd-md5sum`
	
	               	# If we dont have the correct password, give a backup error.
                	if [ "$MD5SUM_CMS" != "$MD5SUM_LOC" ]
                	then
				neonova_backup_check_log "module: md5sum: FAILED! : Incorrect server backup pwd in cms!"
				server_backup_error=1
                	else 
				neonova_backup_check_log "module: md5sum: OK!"
			fi
                else
                        neonova_backup_check_log "module: md5sum: FAILED! : Could not find the md5sum file!"
                        astium_backup_error=1
                fi
	fi

	# Backup check is done for this server.
	neonova_backup_check_log "Finished backup check"

	#
	# Backup is ok, so we create the daily backup archive.
	#
	if [ $server_backup_error -eq 0 ]
	then
		day=`date +%A`
		upload="/home/$serial/upload"
		daily_archive="/home/$serial/archive/daily"

		# Remove old tmp directory.		
		if [ -d $daily_archive/$day.tmp ]
		then
			rm -rf $daily_archive/$day.tmp
		fi

		# Move old directory to tmp directory.	
		if [ -d $daily_archive/$day ]
		then
			mv $daily_archive/$day $daily_archive/$day.tmp
		fi

		# Copy uploaded files to archive.	
		for module in databases logfiles sysinfo system
		do
	
			if [ $module = "databases" ]
			then
				count=`ls $upload/$module/|wc -l`
				
				if [ $count -gt 0 ]
				then
					mkdir -p $daily_archive/$day/$module
					cp -af $upload/$module/* $daily_archive/$day/$module
				fi
			fi

			if [ $module != "databases" ]
			then
				mkdir -p $daily_archive/$day/$module
				cp -af $upload/$module/* $daily_archive/$day/$module
			fi
		done

                # Compress the logfile to safe disk space.
		if [ -d "$daily_archive/$day/logfiles" ]
		then
               		gzip -9 -r $daily_archive/$day/logfiles
		fi

		# Copy uploaded files to archive (using hardlinks, to safe disk space).	
		mkdir -p $daily_archive/$day/data
		cp -alf $upload/data/* $daily_archive/$day/data

		# Copy the backup_pwd md5sum to archive.	
		if [ -d $upload/md5sum ]
		then
			mkdir -p $daily_archive/$day/md5sum
			cp -alf $upload/md5sum/* $daily_archive/$day/md5sum
		fi

		# Remove tmp directory.	
		if [ -d $daily_archive/$day.tmp ]
		then
			rm -rf $daily_archive/$day.tmp
		fi

		# Add backup date (daily archive directory).
		date +%s > $daily_archive/$day/.backup_date

		# Copy extra information.
		test=`ls -all $upload/|grep .backup_client_id`
		if [ -n "$test" ]
		then
			cp $upload/.backup_client_id*	$daily_archive/$day
		fi

		if [ -f $upload/.backup_client_name ]
		then
			cp $upload/.backup_client_name	$daily_archive/$day
		fi

		# Add backup status (daily archive directory).
		echo "OK!" > $daily_archive/$day/.backup_status
	fi

	#
	# Add the backup status to the upload directory.
	#
	if [ $server_backup_error -eq 0 ]
	then
		# Add backup status (upload directory).
		echo "OK!" > $upload/.backup_status

		# Add backup date (upload directory).
		date +%s > $upload/.backup_date
	else 
		# Add backup status (upload directory).
		echo "NOT OK" > $upload/.backup_status
	fi

	#
	# Add server backup status to log/email message.
	#	

	# Get server name.
	name=`cat $NEONOVA_LIST |grep ^$serial|awk -F, '{ print $3 }'`

	if [ $server_backup_error -eq 1 ]
	then
		text="Backup Server $serial : $name : NOT OK! (look in /home/$serial/backup.log for more information)"
		echo $text >> $logfile.not_ok

		text="Backup Server $serial : $name : NOT OK!"
		echo $text >> $logfile_reseller.not_ok

		# Set reseller_backup_error.
		reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"
		echo "1" > $reseller_backup_error

		# Set server backup_error.
		backup_error=1

		# Add backup status to xml file.
		echo "$serial:1" >> $xmlfile
	else
		text="Backup Server $serial : $name : OK"
		echo $text >> $logfile.ok
		echo $text >> $logfile_reseller.ok

		# Add backup status to xml file.
		echo "$serial:0" >> $xmlfile
	fi	
	
done

#
# Create list of servers where the backup is not activated.
#
for serial in `cat $NEONOVA_LIST|grep "[0-9]\{5\}"|grep -v false |grep -v disabled|awk -F, '{ print $1 }'|sort|uniq`
do
        test=`grep "^$serial:" /etc/passwd`
        if [ -z $test ]
        then
		# Get server name.
		name=`cat $NEONOVA_LIST |grep ^$serial|awk -F, '{ print $3 }'`

                text="Backup Server $serial : $name : Not Activated"
		echo $text >> $logfile.not_activated

		# Get reseller id.
		reseller_id=`grep ^$serial $NEONOVA_LIST |awk -F, '{ print $4 }'`

		# Set logfile for the reseller.
		logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"
		echo $text >> $logfile_reseller.not_activated
	
		# Set server backup_error.
                backup_error=1

		# Set reseller_backup_error.
		reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"
		echo "1" > $reseller_backup_error
	fi
done

#
# Create list of servers where the backup is disabled.
#
for serial in `cat $NEONOVA_LIST|grep "[0-9]\{5\}"|grep disabled |awk -F, '{ print $1 }'`
do
	# Get server name.
	name=`cat $NEONOVA_LIST |grep ^$serial|awk -F, '{ print $3 }'`

	# Get reseller id.
	reseller_id=`grep ^$serial $NEONOVA_LIST |awk -F, '{ print $4 }'`

	# Set logfile for the reseller.
	logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"

	text="Backup Server $serial : $name : Backup disabled"
	echo $text >> $logfile.disabled
	echo $text >> $logfile_reseller.disabled
done

#
# Here we send the email to netland of neonova.
#

# Set Subject for server backup mail.
if [ $backup_error -eq 1 ]
then
	subject="Backup report : Some backups failed, Please investigate" 
else 
	subject="Backup report : All backups are ok" 
fi

# Create Email message.
if [ -f $logfile.not_ok ]
then
	cat $logfile.not_ok		>> $logfile
	echo "" 			>> $logfile
fi

DAYOFWEEK=`date +%A`
if [ $DAYOFWEEK = "Monday" ]
then
	if [ -f $logfile.disabled ]
	then
		cat $logfile.disabled	>> $logfile
		echo "" 		>> $logfile
	fi
fi

if [ -f $logfile.not_activated ]
then
	cat $logfile.not_activated	>> $logfile
	echo "" 			>> $logfile
fi

if [ -f $logfile.ok ]
then
	cat $logfile.ok			>> $logfile
	echo "" 			>> $logfile
fi

# Send the backup report email.
cat $logfile|mail $EMAIL -s "$subject"

#
# Here we send the email to all server contacts.
#
for reseller_id in `cat $NEONOVA_RESELLER_LIST |awk -F, '{ print $1 }'|sort|uniq`
do
	# Check if the reseller has a server or sold a server.
	test=`cat $NEONOVA_LIST|awk -F, '{ print  $4 }'|sort|uniq|grep $reseller_id`
	if [ -n "$test" ]
	then

		# reseller_backup_error file.
		reseller_backup_error="$TEMPDIR/reseller-$reseller_id-backup-error"

		# Create empty reseller email.
		logfile_reseller="$TEMPDIR/backup-check-reseller-$reseller_id"
		> logfile_reseller

		# Get backup error.
		backup_error=`cat $reseller_backup_error`

		# Create reseller backup mail.
		reseller_backup_report="$TEMPDIR/reseller-backup-report-$reseller_id.txt"
		> $reseller_backup_report

		#
		# Adding Subject to the reseller email.	
		#
		if [ $backup_error -eq 1 ]
		then
			subject="Server backup report: Some backups failed!"
			echo "Subject: $subject" >> $reseller_backup_report
		else
     		 	subject="Server backup report: All backups are OK!"
			echo "Subject: $subject" >> $reseller_backup_report
		fi

		#
		# Adding email adressen to the reseller email.
		#
		for email_adres in `grep ^$reseller_id, $NEONOVA_RESELLER_LIST|awk -F, '{ print $2 }'`
		do
			echo "To:$email_adres" 			>> $reseller_backup_report
		done
		#echo "Bcc:perry@couprie.net"  			>> $reseller_backup_report

		# Add message header.
		echo "" 					>> $reseller_backup_report
		add_backup_mail_header 				   $reseller_backup_report

		if [ -f $logfile_reseller.not_ok ]
		then
			echo "" 				>> $reseller_backup_report
			cat $logfile_reseller.not_ok		>> $reseller_backup_report
		fi

		if [ -f $logfile_reseller.not_activated ]
		then
			echo "" 				>> $reseller_backup_report
			cat $logfile_reseller.not_activated	>> $reseller_backup_report
		fi

		if [ -f $logfile_reseller.disabled ]
		then
			echo "" 				>> $reseller_backup_report
			cat $logfile_reseller.disabled		>> $reseller_backup_report
		fi

		if [ -f $logfile_reseller.ok ]
		then
			echo "" 				>> $reseller_backup_report
			cat $logfile_reseller.ok		>> $reseller_backup_report
		fi

		# Add mail footer.
		echo "" 					>> $reseller_backup_report
		add_backup_mail_footer 				   $reseller_backup_report

		# Send the email we created to the reseller.
		cat $reseller_backup_report|/usr/bin/unix2dos|/usr/sbin/sendmail -fbeheer@neonova.nl -F"NeoNova Beheer" -t
	fi
done 

# Remove temp files left by the daily rsync processes.
/usr/local/neonova/sbin/nnb-remove-tmp-files

# Create a backup list for each astium. with all the backup that are in the daily and weekly archive.
/usr/local/neonova/sbin/nnb-create-backup-lists

# Create softlinks to all backups.
/usr/local/neonova/sbin/nnb-create-softlinks

# Call post backup script.
if [ -x /usr/local/neonova/sbin/neonova-backup-post.sh ]
then
	. /usr/local/neonova/sbin/neonova-backup-post.sh
fi

#
# Create weekly archive (copy the Sundat daily archive to the weekly archive).
#
for serial in `cat $NEONOVALIST_LOCAL`
do
	# Get day of the week.	
	today=`date +%A`

	# Calculate weekly backup number (1,2,3 or 4).
	week_nr=`date +%V`
	a=`expr $week_nr % $WEEKLY_BACKUP_COUNT`
	b=`expr $a + 1`
	week_nr=`printf "%02d" $b`

	daily_archive="/home/$serial/archive/daily"
	weekly_archive="/home/$serial/archive/weekly/$week_nr"

	# Hack for weekly archive om seperate disk.
	weekly_archive="$WEEKLY_ARCHIVE_DIR/$serial/archive/weekly/$week_nr"

	#
	# Its not the and of the week, but copy some data to the weekly archive, 
	# Only if the weekly archive directory does not exists yet.
	# This saves some time at the end of the week.
	#
	if [ ! $today = 'Sunday' ]
	then	
		# Set source and target directory.
		source="$daily_archive/$today/data/"
		target="$weekly_archive/Sunday/data"

		# Check if data directory exists.
		if [ -d $source -a ! -d $target ]
		then
			# Time now.
			time_now=`date +%s`
	
			# Time the backup archive has been running.
			let "time_div = $time_now - $time_start"
	
			# If the backup archive has been running, less then 6 hours 
			# start pre seeding the not existing weekly archive.
			if [ $time_div -lt 21600 ]
			then
		
				# Target does not exists so create it.	
				if [ ! -d $target ]
				then
					mkdir -p $target
				fi
	
				# Here we copy some data to the weekly archive.
    				TEMP_RSYNC_DIR="/mnt/dsk2/week_backup/.tmp"
				mkdir -p $TEMP_RSYNC_DIR
				rsync -H -avz --temp-dir=$TEMP_RSYNC_DIR --progress --hard-links --delete $source $target
			fi
		fi
	fi

	#
	# Create weekly archive directory by means of rsync and using hardlinks to save space.
	#
	if [ $today = 'Sunday' ]
	then
		# Set backup log for a specific server.
		backup_log_weekly_archive="/home/$serial/backup-weekly.log"

		# If log file does not exists, create it.
		if [ -f $backup_log_weekly_archive ]
		then
			touch $backup_log_weekly_archive
		fi

		# Log start weekly backup.	
		date=`date`
		echo "Neonova Weekly Backup: Start : $date" >> $backup_log_weekly_archive

		# Reset exit_status.
		exit_status=1

		if [ ! -d $weekly_archive/$today ]
		then
			mkdir -p $weekly_archive/$today
		fi

		if [ -d $daily_archive ]
		then

                        # Create tmp rsync directory if it does not exists yet. 
    			TEMP_RSYNC_DIR="/mnt/dsk2/week_backup/.tmp"
                        if [ ! -d "$TEMP_RSYNC_DIR" ]
                        then
                                mkdir -p $TEMP_RSYNC_DIR
                        fi

			source="$daily_archive/$today/"
			target="$weekly_archive/$today"
			rsync -H -avz --temp-dir=$TEMP_RSYNC_DIR --progress --hard-links --delete $source $target
		fi

                # On error set exit_status.
                if [ $? -ne 0 ]
                then
                        exit_status=1
                        exit_message="1:weekly_archive:$week_nr:rsync to weekly archive failed."
		else 
                        exit_status=0
                        exit_message="0:weekly_archive:$week_nr:rsync to weekly archive OK!."
		fi

		# Add exit status to logfile.
		echo $exit_message >> $backup_log_weekly_archive

		# Log End of weekly backup.	
		date=`date`
		echo "Neonova Weekly Backup: End : $date" >> $backup_log_weekly_archive
		echo " " >> $backup_log_weekly_archive
	fi
done

# Remove temp working directory.
#remove_temp_dir

# Remove lock file.
removelockfile-backup-archive

