#!/usr/bin/env bash PROGRAM="backup_tool_script" PROGRAM_VERSION=0.5.0 PROGRAM_BUILD=2019081901 AUTHOR="(C) 2017-2019 by Orsiris de Jong" CONTACT="http://www.netpower.fr - ozy@netpower.fr" IS_STABLE=true ## backup_tool_script - A script to check @name@ backup sanity ## backup_tool_script can verify a given number of backups for each client. It can run verifiy operations in parallel. ## Verify operations are timed in order to stop them after a given amount of time, leaving the system performance ready for backup operations. ## The script can also list clients that have outdated backups. It uses two different methods to list clients in order to detect rogue clients. ## It can also ensure that the @name@ server service is running properly, relaunch it if needed, on a scheduled basis. ## The script can send a warning / error when problems are found, even while operating. ## The script can send a warning / error when disk quotas exceed, even while operating. ## backup_tool_script can also launch vss_strip for each file found in a given directory. ## The script can also send mails directly to the client, if an email address is given in client config file as 'label = email_address : some@example.com' ## When exiting, backup_tool_script ensures that no forked @name@ processes remain, without touching other @name@ processes that didn't belong to backup_tool_script. ## Set an unique identifier for the script which will be used for logs and alert mails INSTANCE_ID="base" ## Backup verifications timers ## After how much time (in seconds) for a single verification a warning should be logged (defaults to 3 hours) SOFT_MAX_EXEC_TIME_PER_VERIFY=10800 ## After how much time (in seconds) for a single verification the process should be stopped (defaults to 5 hours) HARD_MAX_EXEC_TIME_PER_VERIFY=64800 ## After how much seconds of execution of all steps a warning should be logged (defaults to 10 hours) SOFT_MAX_EXEC_TIME=64800 ## After how much seconds of execution of all steps a verification process should be stopped (defaults to 12 hours) HARD_MAX_EXEC_TIME=82800 # Verify operations checks ## When a client isn't idle, we can postpone the it's backup verification process. How many times should we retry the verification command. Set this to 0 to disable operation postponing POSTPONE_RETRY=2 ## When postponed, how much time (in seconds) before next try (defauls to 1 hour) POSTPONE_TIME=3600 ## Backup executable (can be set to /usr/sbin/@name@, /usr/local/sbin/@name@, or autodetect via $(type -p @name@)) BACKUP_EXECUTABLE=@sbindir@/@name@ # PROD = @sbindir@/@name@ ## @name@ service type (can be "initv" or "systemd") SERVICE_TYPE=initv # PROD = initv ## How many simultaneous verify operations should be launched (please check I/O and CPU usage before increasing this) PARELLEL_VERIFY_CONCURRENCY=2 # ------------ Mail alert settings ------------- ## General alert mail subject MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." ## Optional change of mail body encoding (using iconv) ## By default, all mails are sent in UTF-8 format without headers ## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle) for maximal compatibility MAIL_BODY_CHARSET="ISO-8859-1" # ------------ Client specific alert email template -------------- ## Email subject CLIENT_ALERT_SUBJECT="@name@ backup - Warning about your backup" ## Valid message body placeholders are ## [NUMBERDAYS] is the number of days we check. ## [QUOTAEXCEED] is the size of the actual backup versus the quota. ## [CLIENT] is the client name ## [INSTANCE] is the current instance name ## Message sent directly to client email address when no recent backups are found. CLIENT_ALERT_BODY_OUTDATED="Hello, No valid backup sets found in the last [NUMBERDAYS] day(s) for client [CLIENT]. Please leave your computer online enough time for a backup to occur. Contact your system administrator for further information. @name@ backup server [INSTANCE]. " # Message sent directly to client email address when quota exceeded. CLIENT_ALERT_BODY_QUOTA="Hello, Your backup disk quota has been exceeded ([QUOTAEXCEED]) for client [CLIENT]. Contact your system administrator for further information. @name@ backup server [INSTANCE]. " # ------------ Do not modify under this line unless you have great cow powers -------------- if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 fi export LC_ALL=C _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags ERROR_ALERT=false WARN_ALERT=false LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) SCRIPT_PID=$$ # Get a random number on Windows BusyBox alike, also works on most Unixes that have dd, if dd is not found, then return $RANDOM # Get a random number of digits length on Windows BusyBox alike, also works on most Unixes that have dd function PoorMansRandomGenerator { local digits="${1}" # The number of digits to generate local number # Some read bytes can't be used, se we read twice the number of required bytes dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do number=$number$(printf "%d" "'$char") if [ ${#number} -ge $digits ]; then echo ${number:0:$digits} break; fi done } # Initial TSTMAP value before function declaration TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5) ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" ## Default log file until config file is loaded if [ -w /var/log ]; then LOG_FILE="/var/log/$PROGRAM.log" elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then LOG_FILE="$HOME/$PROGRAM.log" elif [ -w . ]; then LOG_FILE="./$PROGRAM.log" else LOG_FILE="/tmp/$PROGRAM.log" fi ## Default directory where to store temporary run files if [ -w /tmp ]; then RUN_DIR=/tmp elif [ -w /var/tmp ]; then RUN_DIR=/var/tmp else RUN_DIR=. fi #### DEBUG SUBSET #### ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG _DEBUG=false #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then _DEBUG=false _LOGGER_VERBOSE=false else trap 'TrapError ${LINENO} $?' ERR _LOGGER_VERBOSE=true fi if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console SLEEP_TIME=.1 fi #### DEBUG SUBSET END #### #### Logger SUBSET #### #### RemoteLogger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array function joinString { local IFS="$1"; shift; echo "$*"; } # Sub function of Logger function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment if [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") else echo -e "$stdValue" fi fi } # General log function with log levels: # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr # _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels # Except for VERBOSE, all loglevels are ALWAYS sent to log file # CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged # NOTICE sent to stdout # VERBOSE sent to stdout if _LOGGER_VERBOSE=true # ALWAYS is sent to stdout unless _LOGGER_SILENT=true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=true function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="$(date '+%Y-%m-%d %H:%M:%S') - " else prefix="" fi if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then _Logger "$prefix$value" "$prefix$value" fi return elif [ "$level" == "VERBOSE" ]; then if [ $_LOGGER_VERBOSE == true ]; then _Logger "$prefix($level):$value" "$prefix$value" fi return elif [ "$level" == "ALWAYS" ]; then _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == true ]; then _Logger "$prefix$value" "$prefix$value" return fi elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { local pid="${1}" # Parent pid to kill childs local self="${2:-false}" # Should parent be killed too ? # Paranoid checks, we can safely assume that $pid should not be 0 nor 1 if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then Logger "Bogus pid given [$pid]." "CRITICAL" return 1 fi if kill -0 "$pid" > /dev/null 2>&1; then if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" children="${children/$pid/}" fi for child in $children; do Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG KillChilds "$child" true done fi fi # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if [ "$self" == true ]; then # We need to check for pid again because it may have disappeared after recursive function call if kill -0 "$pid" > /dev/null 2>&1; then kill -s TERM "$pid" Logger "Sent SIGTERM to process [$pid]." "DEBUG" if [ $? -ne 0 ]; then sleep 15 Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" kill -9 "$pid" if [ $? -ne 0 ]; then Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" return 1 fi # Simplify the return 0 logic here else return 0 fi else return 0 fi else return 0 fi } function KillAllChilds { local pids="${1}" # List of parent pids to kill separated by semi-colon local self="${2:-false}" # Should parent be killed too ? __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local errorcount=0 IFS=';' read -a pidsArray <<< "$pids" for pid in "${pidsArray[@]}"; do KillChilds $pid $self if [ $? -ne 0 ]; then errorcount=$((errorcount+1)) fi done return $errorcount } # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending function SendAlert { local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run local attachment="${2:-true}" # Should we send the log file as attachment __CheckArguments 0-2 $# "$@" #__WITH_PARANOIA_DEBUG local attachmentFile local subject local body if [ "$DESTINATION_MAILS" == "" ]; then return 0 fi if [ "$_DEBUG" == true ]; then Logger "Debug mode, no warning mail will be sent." "NOTICE" return 0 fi if [ $attachment == true ]; then attachmentFile="$LOG_FILE" if type "$COMPRESSION_PROGRAM" > /dev/null 2>&1; then eval "cat \"$LOG_FILE\" \"$COMPRESSION_PROGRAM\" > \"$ALERT_LOG_FILE\"" if [ $? -eq 0 ]; then attachmentFile="$ALERT_LOG_FILE" fi fi fi body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" elif [ $WARN_ALERT == true ]; then subject="Warning alert for $INSTANCE_ID" else subject="Alert for $INSTANCE_ID" fi if [ $runAlert == true ]; then subject="Currently runing - $subject" else subject="Finished run - $subject" fi SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD" # Delete tmp log file if [ "$attachment" == true ]; then if [ -f "$ALERT_LOG_FILE" ]; then rm -f "$ALERT_LOG_FILE" fi fi } # Generic email sending function. # Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or "" # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" # Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend) # attachment is optional but must be in windows format like "c:\\some\path\\my.file", or "" # smtp_server.domain.tld is mandatory, as is smtpPort (should be 25, 465 or 587) # encryption can be set to tls, ssl or none # smtpUser and smtpPassword are optional # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" # If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc #set ttycharset=iso-8859-1 #set sendcharsets=iso-8859-1 #set encoding=8bit function SendEmail { local subject="${1}" local message="${2}" local destinationMails="${3}" local attachment="${4}" local senderMail="${5}" local smtpServer="${6}" local smtpPort="${7}" local encryption="${8}" local smtpUser="${9}" local smtpPassword="${10}" __CheckArguments 3-10 $# "$@" #__WITH_PARANOIA_DEBUG local mail_no_attachment= local attachment_command= local encryption_string= local auth_string= local i if [ "${destinationMails}" != "" ]; then for i in "${destinationMails[@]}"; do if [ $(CheckRFC822 "$i") -ne 1 ]; then Logger "Given email [$i] does not seem to be valid." "WARN" fi done else Logger "No valid email addresses given." "WARN" return 1 fi # Prior to sending an email, convert its body if needed if [ "$MAIL_BODY_CHARSET" != "" ]; then if type iconv > /dev/null 2>&1; then echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP" message="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP")" else Logger "iconv utility not installed. Will not convert email charset." "NOTICE" fi fi if [ ! -f "$attachment" ]; then attachment_command="-a $attachment" mail_no_attachment=1 else mail_no_attachment=0 fi if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then if [ "$smtpPort" == "" ]; then Logger "Missing smtp port, assuming 25." "WARN" smtpPort=25 fi if type sendmail > /dev/null 2>&1; then if [ "$encryption" == "tls" ]; then echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" elif [ "$encryption" == "ssl" ]; then echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" elif [ "$encryption" == "none" ]; then echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" else echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" Logger "Bogus email encryption used [$encryption]." "WARN" fi if [ $? -ne 0 ]; then Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN" # Do not bother try other mail systems with busybox return 1 else return 0 fi else Logger "Sendmail not present. Will not send any mail" "WARN" return 1 fi fi if type mutt > /dev/null 2>&1 ; then # We need to replace spaces with comma in order for mutt to be able to process multiple destinations echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command if [ $? -ne 0 ]; then Logger "Cannot send mail via $(type -p mutt) !!!" "WARN" else Logger "Sent mail using mutt." "NOTICE" return 0 fi fi if type mail > /dev/null 2>&1 ; then # We need to detect which version of mail is installed if ! $(type -p mail) -V > /dev/null 2>&1; then # This may be MacOS mail program attachment_command="" elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then attachment_command="-A $attachment" elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then attachment_command="-a$attachment" else attachment_command="" fi echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" if [ $? -ne 0 ]; then Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" echo "$message" | $(type -p mail) -s "$subject" "$destinationMails" if [ $? -ne 0 ]; then Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN" else Logger "Sent mail using mail command without attachment." "NOTICE" return 0 fi else Logger "Sent mail using mail command." "NOTICE" return 0 fi fi if type sendmail > /dev/null 2>&1 ; then echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destinationMails" if [ $? -ne 0 ]; then Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN" else Logger "Sent mail using sendmail command without attachment." "NOTICE" return 0 fi fi # Windows specific if type "mailsend.exe" > /dev/null 2>&1 ; then if [ "$senderMail" == "" ]; then Logger "Missing sender email." "ERROR" return 1 fi if [ "$smtpServer" == "" ]; then Logger "Missing smtp port." "ERROR" return 1 fi if [ "$smtpPort" == "" ]; then Logger "Missing smtp port, assuming 25." "WARN" smtpPort=25 fi if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then Logger "Bogus smtp encryption, assuming none." "WARN" encryption_string= elif [ "$encryption" == "tls" ]; then encryption_string=-starttls elif [ "$encryption" == "ssl" ]:; then encryption_string=-ssl fi if [ "$smtpUser" != "" ] && [ "$smtpPassword" != "" ]; then auth_string="-auth -user \"$smtpUser\" -pass \"$smtpPassword\"" fi $(type mailsend.exe) -f "$senderMail" -t "$destinationMails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtpServer" -port "$smtpPort" $encryption_string $auth_string if [ $? -ne 0 ]; then Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" else Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" return 0 fi fi # pfSense specific if [ -f /usr/local/bin/mail.php ]; then echo "$message" | /usr/local/bin/mail.php -s="$subject" if [ $? -ne 0 ]; then Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" else Logger "Sent mail using pfSense mail.php." "NOTICE" return 0 fi fi # If function has not returned 0 yet, assume it is critical that no alert can be sent Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue } #### TrapError SUBSET #### function TrapError { local job="$0" local line="$1" local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } #### TrapError SUBSET END #### # Quick and dirty performance logger only used for debugging function _PerfProfiler { #__WITH_PARANOIA_DEBUG local perfString #__WITH_PARANOIA_DEBUG local i #__WITH_PARANOIA_DEBUG #__WITH_PARANOIA_DEBUG perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG #__WITH_PARANOIA_DEBUG for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG done #__WITH_PARANOIA_DEBUG #__WITH_PARANOIA_DEBUG if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG #__WITH_PARANOIA_DEBUG Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG } # Checks email address validity function CheckRFC822 { local mail="${1}" local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" if [[ $mail =~ $rfc822 ]]; then echo 1 else echo 0 fi } # Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" if type expr > /dev/null 2>&1; then expr "$value" : '^[0-9]\{1,\}$' > /dev/null 2>&1 if [ $? -eq 0 ]; then echo 1 else echo 0 fi else if [[ $value =~ ^[0-9]+$ ]]; then echo 1 else echo 0 fi fi } # Usage [ $(IsNumeric $var) -eq 1 ] function IsNumeric { local value="${1}" if type expr > /dev/null 2>&1; then expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1 if [ $? -eq 0 ]; then echo 1 else echo 0 fi else if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then echo 1 else echo 0 fi fi } function IsNumericExpand { eval "local value=\"${1}\"" # Needed eval so variable variables can be processed echo $(IsNumeric "$value") } ## Modified version of http://stackoverflow.com/a/8574392 ## Usage: [ $(ArrayContains "needle" "${haystack[@]}") -eq 1 ] function ArrayContains () { local needle="${1}" local haystack="${2}" local e if [ "$needle" != "" ] && [ "$haystack" != "" ]; then for e in "${@:2}"; do if [ "$e" == "$needle" ]; then echo 1 return fi done fi echo 0 return } _OFUNCTIONS_SPINNER="|/-\\" function Spinner { if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then return 0 else printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER" _OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???} return 0 fi } ## Main asynchronous execution function ## Function can work in: ## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for ## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed. ## Example of improved wait $! ## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false ## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds ## sleep 15 & ## pid=$! ## sleep 20 & ## pid2=$! ## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2" ## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel. ## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr" ## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]" ## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions" ## Bear in mind that given commands and conditions need to be quoted ## ExecTasks has the following ofunctions subfunction requirements: ## Spinner ## Logger ## JoinString ## KillChilds ## Full call ##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes function ExecTasks { # Mandatory arguments local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands # Optional arguments local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) # Parallelism specific arguments local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors __CheckArguments 1-18 $# "$@" local i Logger "${FUNCNAME[0]} id [$id] called by [${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} < ${FUNCNAME[5]} < ${FUNCNAME[6]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG #__WITH_PARANOIA_DEBUG # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode if [ "$_DEBUG" == true ]; then declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll) for i in "${booleans[@]}"; do test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" eval "$test" done declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries) for i in "${integers[@]}"; do test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" eval "$test" done fi # Expand validExitCodes into array IFS=';' read -r -a validExitCodes <<< "$validExitCodes" # ParallelExec specific variables local auxItemCount=0 # Number of conditional commands local commandsArray=() # Array containing commands local commandsConditionArray=() # Array containing conditional commands local currentCommand # Variable containing currently processed command local currentCommandCondition # Variable containing currently processed conditional command local commandsArrayPid=() # Array containing commands indexed by pids local commandsArrayOutput=() # Array containing command results indexed by pids local postponedRetryCount=0 # Number of current postponed commands retries local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedCounter=0 local isPostponedCommand=false # Is the current command from a postponed file ? local postponedExecTime=0 # How much time has passed since last postponed condition was checked local needsPostponing # Does currentCommand need to be postponed local temp # Common variables local pid # Current pid working on local pidState # State of the process local mainItemCount=0 # number of given items (pids or commands) local readFromFile # Should we read pids / commands from a file (true) local counter=0 local log_ttime=0 # local time instance for comparaison local seconds_begin=$SECONDS # Seconds since the beginning of the script local exec_time=0 # Seconds since the beginning of this function local retval=0 # return value of monitored pid process local subRetval=0 # return value of condition commands local errorcount=0 # Number of pids that finished with errors local pidsArray # Array of currently running pids local newPidsArray # New array of currently running pids for next iteration local pidsTimeArray # Array containing execution begin time of pids local executeCommand # Boolean to check if currentCommand can be executed given a condition local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local functionMode local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3) local randomOutputName # Random filename for command outputs local currentRunningPids # String of pids running, used for debugging purposes only # Initialise global variable eval "WAIT_FOR_TASK_COMPLETION_$id=\"\"" eval "HARD_MAX_EXEC_TIME_REACHED_$id=false" # Init function variables depending on mode if [ $numberOfProcesses -gt 0 ]; then functionMode=ParallelExec else functionMode=WaitForTaskCompletion fi if [ $readFromFile == false ]; then if [ $functionMode == "WaitForTaskCompletion" ]; then IFS=';' read -r -a pidsArray <<< "$mainInput" mainItemCount="${#pidsArray[@]}" else IFS=';' read -r -a commandsArray <<< "$mainInput" mainItemCount="${#commandsArray[@]}" IFS=';' read -r -a commandsConditionArray <<< "$auxInput" auxItemCount="${#commandsConditionArray[@]}" fi else if [ -f "$mainInput" ]; then mainItemCount=$(wc -l < "$mainInput") readFromFile=true else Logger "Cannot read main file [$mainInput]." "WARN" fi if [ "$auxInput" != "" ]; then if [ -f "$auxInput" ]; then auxItemCount=$(wc -l < "$auxInput") else Logger "Cannot read aux file [$auxInput]." "WARN" fi fi fi if [ $functionMode == "WaitForTaskCompletion" ]; then # Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode counter=$mainItemCount fi Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { if [ $spinner == true ]; then Spinner fi if [ $counting == true ]; then exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi if [ $keepLogging -ne 0 ]; then # This log solely exists for readability purposes before having next set of logs if [ ${#pidsArray[@]} -eq $numberOfProcesses ] && [ $log_ttime -eq 0 ]; then log_ttime=$exec_time Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" fi if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second log_ttime=$exec_time if [ $functionMode == "WaitForTaskCompletion" ]; then Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" elif [ $functionMode == "ParallelExec" ]; then Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" fi fi fi fi if [ $exec_time -gt $softMaxTime ]; then if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN" softAlert=true SendAlert true false fi fi if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then if [ $noTimeErrorLog != true ]; then Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" fi for pid in "${pidsArray[@]}"; do KillChilds $pid true if [ $? -eq 0 ]; then Logger "Task with pid [$pid] stopped successfully." "NOTICE" else if [ $noErrorLogsAtAll != true ]; then Logger "Could not stop task with pid [$pid]." "ERROR" fi fi errorcount=$((errorcount+1)) done if [ $noTimeErrorLog != true ]; then SendAlert true false fi eval "HARD_MAX_EXEC_TIME_REACHED_$id=true" if [ $functionMode == "WaitForTaskCompletion" ]; then return $errorcount else return 129 fi fi } function _ExecTasksPidsCheck { newPidsArray=() if [ "$currentRunningPids" != "$(joinString " " ${pidsArray[@]})" ]; then Logger "ExecTask running for pids [$(joinString " " ${pidsArray[@]})]." "DEBUG" currentRunningPids="$(joinString " " ${pidsArray[@]})" fi for pid in "${pidsArray[@]}"; do if [ $(IsInteger $pid) -eq 1 ]; then if kill -0 $pid > /dev/null 2>&1; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then # Check if pid hasn't run more than soft/hard perProcessTime pidsTimeArray[$pid]=$((SECONDS - seconds_begin)) if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then Logger "Max soft execution time [$softPerProcessTime] exceeded for pid [$pid]." "WARN" if [ "${commandsArrayPid[$pid]}]" != "" ]; then Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN" fi softAlert=true SendAlert true false fi fi if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR" if [ "${commandsArrayPid[$pid]}]" != "" ]; then Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN" fi fi KillChilds $pid true if [ $? -eq 0 ]; then Logger "Command with pid [$pid] stopped successfully." "NOTICE" else if [ $noErrorLogsAtAll != true ]; then Logger "Could not stop command with pid [$pid]." "ERROR" fi fi errorcount=$((errorcount+1)) if [ $noTimeErrorLog != true ]; then SendAlert true false fi fi newPidsArray+=($pid) fi else # pid is dead, get its exit code from wait command wait $pid retval=$? # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi if [ -f "${commandsArrayOutput[$pid]}" ]; then Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "ERROR" fi fi errorcount=$((errorcount+1)) # Welcome to variable variable bash hell if [ "$failedPidsList" == "" ]; then failedPidsList="$pid:$retval" else failedPidsList="$failedPidsList;$pid:$retval" fi else Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" fi fi hasPids=true ##__WITH_PARANOIA_DEBUG fi done # hasPids can be false on last iteration in ParallelExec mode if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime if [ "$_PERF_PROFILER" == true ]; then ##__WITH_PARANOIA_DEBUG _PerfProfiler ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG } while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do _ExecTasksTimeCheck retval=$? if [ $retval -ne 0 ]; then return $retval; fi # The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them if [ $functionMode == "ParallelExec" ]; then while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do _ExecTasksTimeCheck retval=$? if [ $retval -ne 0 ]; then return $retval; fi executeCommand=false isPostponedCommand=false currentCommand="" currentCommandCondition="" needsPostponing=false if [ $readFromFile == true ]; then # awk identifies first line as 1 instead of 0 so we need to increase counter currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput") if [ $auxItemCount -ne 0 ]; then currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput") fi # Check if we need to fetch postponed commands if [ "$currentCommand" == "" ]; then currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP") currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP") isPostponedCommand=true fi else currentCommand="${commandsArray[$counter]}" if [ $auxItemCount -ne 0 ]; then currentCommandCondition="${commandsConditionArray[$counter]}" fi if [ "$currentCommand" == "" ]; then currentCommand="${postponedCommandsArray[$postponedCounter]}" currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}" isPostponedCommand=true fi fi # Check if we execute postponed commands, or if we delay them if [ $isPostponedCommand == true ]; then # Get first value before '@' postponedExecTime="${currentCommand%%@*}" postponedExecTime=$((SECONDS-postponedExecTime)) # Get everything after first '@' temp="${currentCommand#*@}" # Get first value before '@' postponedRetryCount="${temp%%@*}" # Replace currentCommand with actual filtered currentCommand currentCommand="${temp#*@}" # Since we read a postponed command, we may decrase postponedItemCounter postponedItemCount=$((postponedItemCount-1)) #Since we read one line, we need to increase the counter postponedCounter=$((postponedCounter+1)) else postponedRetryCount=0 postponedExecTime=0 fi if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $minTimeBetweenRetries ]) || [ $isPostponedCommand == false ]; then if [ "$currentCommandCondition" != "" ]; then Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG" eval "$currentCommandCondition" & ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true subRetval=$? if [ $subRetval -ne 0 ]; then # is postponing enabled ? if [ $maxPostponeRetries -gt 0 ]; then Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE" postponedRetryCount=$((postponedRetryCount+1)) if [ $postponedRetryCount -ge $maxPostponeRetries ]; then Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE" else needsPostponing=true fi postponedExecTime=0 else Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE" fi else executeCommand=true fi else executeCommand=true fi else needsPostponing=true fi if [ $needsPostponing == true ]; then postponedItemCount=$((postponedItemCount+1)) if [ $readFromFile == true ]; then echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" else postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand") postponedCommandsConditionArray+=("$currentCommandCondition") fi fi if [ $executeCommand == true ]; then Logger "Running command [$currentCommand]." "DEBUG" randomOutputName=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5) eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 & pid=$! pidsArray+=($pid) commandsArrayPid[$pid]="$currentCommand" commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" # Initialize pid execution time array pidsTimeArray[$pid]=0 else Logger "Skipping command [$currentCommand]." "DEBUG" fi if [ $isPostponedCommand == false ]; then counter=$((counter+1)) fi _ExecTasksPidsCheck done fi _ExecTasksPidsCheck done Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Return exit code if only one process was monitored, else return number of errors # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\"" if [ $mainItemCount -eq 1 ]; then return $retval else return $errorcount fi } function CleanUp { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$_DEBUG" != true ]; then rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP" # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi } #__BEGIN_WITH_PARANOIA_DEBUG function __CheckArguments { # Checks the number of arguments of a function and raises an error if some are missing if [ "$_DEBUG" == true ]; then local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments local numberOfGivenArguments="${2}" # Number of arguments that have been passed local minArgs local maxArgs # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) # If any of the arguments contains spaces, bash things there are two aguments # In order to avoid this, we need to iterate over ${3} and count callerName="${FUNCNAME[1]}" local iterate=3 local fetchArguments=true local argList="" local countedArguments while [ $fetchArguments == true ]; do cmd='argument=${'$iterate'}' eval $cmd if [ "$argument" == "" ]; then fetchArguments=false else argList="$argList[Argument $((iterate-2)): $argument] " iterate=$((iterate+1)) fi done countedArguments=$((iterate-3)) if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then minArgs=$numberOfArguments maxArgs=$numberOfArguments else IFS='-' read minArgs maxArgs <<< "$numberOfArguments" fi Logger "Entering function [$callerName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" Logger "$callerName arguments: $argList" "ERROR" else if [ ! -z "$argList" ]; then Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" fi fi fi } #__END_WITH_PARANOIA_DEBUG ############################################ END OF OFUNCTIONS CODE function TrapQuit { local exitcode # Get ERROR / WARN alert flags from subprocesses that call Logger if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true fi if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true fi if [ $ERROR_ALERT == true ]; then Logger "$PROGRAM finished with errors." "ERROR" if [ "$_DEBUG" != true ] then SendAlert false false else Logger "Debug mode, no alert mail will be sent." "NOTICE" fi exitcode=1 elif [ $WARN_ALERT == true ]; then Logger "$PROGRAM finished with warnings." "WARN" if [ "$_DEBUG" != true ] then SendAlert false false else Logger "Debug mode, no alert mail will be sent." "NOTICE" fi exitcode=2 # Warning exit code must not force daemon mode to quit else Logger "$PROGRAM finished." "ALWAYS" exitcode=0 fi CleanUp KillChilds $SCRIPT_PID > /dev/null 2>&1 Logger "Elapsed [$SECONDS] seconds." "DEBUG" exit $exitcode } # Takes as many file arguments as needed function InterleaveFiles { local counter=0 local hasLine=true local i while [ $hasLine == true ]; do hasLine=false for i in "$@"; do line=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$i") if [ -n "$line" ]; then echo "$line" hasLine=true fi done counter=$((counter+1)) done } function ListClients { local backupDir="${1}" local configFile="${2}" local clientConfDir="${3}" __CheckArguments 0-3 $# "$@" #__WITH_PARANOIA_DEBUG local clientIsIncluded local clientIsExcluded local excludeArray local client local clientEmail local configString local i local j if [ -f "$configFile" ]; then configString="-c \"$configFile\"" fi if [ "$clientConfDir" == "" ]; then clientConfDir="/etc/@name@/clientconfdir" fi if [ -d "$backupDir" ]; then # File 'backup_stats' is there only when a backup is finished find "$backupDir" -mindepth 3 -maxdepth 3 -type f -name "backup_stats" | grep -e '.*' > /dev/null if [ $? -ne 0 ]; then Logger "The directory [$backupDir] does not seem to be a @name@ folder. Please check the path. Additionnaly, protocol 2 directores need to specify the dedup group directory and the client subfolder." "ERROR" Logger "This message may also show if the current user running this script does not have sufficient privileges on the @name@ folder." "ERROR" fi fi # Autodetect clients or use provided client with --client or -C option if [ "$GIVEN_CLIENT" == "" ]; then # Using both @name@ -a S list and find method in order to find maximum backup clients cmd="$BACKUP_EXECUTABLE $configString -a S | grep \"last backup\" | awk '{print \$1}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP\"" Logger "Running cmd [$cmd]." "DEBUG" eval "$cmd" & ExecTasks $! "${FUNCNAME[0]}_lastbackup" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING if [ $? -ne 0 ]; then Logger "Enumerating @name@ clients via [$BACKUP_EXECUTABLE $configString -a S] failed. Check provided @name@ client config file." "ERROR" else Logger "@name@ detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP)" "DEBUG" fi # First exp removes everything before last '/' # sed tested on linux and BSD (should work on MacOS too) if [ -d "$backupDir" ]; then find "$backupDir" -mindepth 1 -maxdepth 1 -type d | sed -e "s/\(.*\)\/\(.*\)/\2/g" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP" while IFS=$'\n' read -r client; do find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | grep -e '.*' > /dev/null 2>&1 if [ $? -eq 0 ]; then echo "$client" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" fi done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP" fi if [ ! -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" ]; then touch "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" fi Logger "Backup file detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG" # Merge all clients found by @name@ executable and manual check sort "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" | uniq > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP" else echo "$GIVEN_CLIENT" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP" fi while IFS=$'\n' read -r client; do clientIsIncluded=false clientIsExcluded=false IFS=',' read -a includeArray <<< "$INCLUDE_CLIENTS" for i in "${includeArray[@]}"; do echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1 if [ $? -eq 0 ]; then clientIsIncluded=true fi done IFS=',' read -a excludeArray <<< "$EXCLUDE_CLIENTS" for i in "${excludeArray[@]}"; do echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1 if [ $? -eq 0 ]; then clientIsExcluded=true fi done if ([ $clientIsIncluded == false ] && [ $clientIsExcluded == true ]); then Logger "Ommiting client [$client]." "NOTICE" else if [ -f "$backupDir$client/current/timestamp" ]; then Logger "Found client [$client] backup directory." "NOTICE" elif [ -d "$backupDir" ]; then Logger "Client [$client] does not seem to have any backups (check provided backup directory)." "WARN" fi CLIENT_LIST+=("$client") # Client email is a label in client config file # Check whether we can fetch the default value if [ -f "$clientConfDir/$client" ]; then clientEmail=$(egrep "^label( )?=( )?email_address( )?:( )?" "$clientConfDir/$client") if [ "$clientEmail" != "" ]; then clientEmail="${clientEmail#*:}" # Remove eventual spaces clientEmail="${clientEmail/ /}" for j in $clientEmail; do if [ $(CheckRFC822 "$j") -eq 1 ]; then if [ "${CLIENT_EMAIL["$client"]}" == "" ]; then CLIENT_EMAIL["$client"]="$j" else CLIENT_EMAIL["$client"]=${CLIENT_EMAIL["$client"]}" $j" fi else Logger "Client [$client] has a bogus mail address [$j]." "WARN" fi done else Logger "Client [$client] has no mail address set." "NOTICE" fi elif ([ "$CLIENTS_ALERT_QUOTAS" == true ] || [ "$CLIENTS_ALERT_OUTDATED" == true ]); then Logger "Cannot find client config file [$clientConfDir/$client] to fetch email address." "ERROR" fi fi done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP" } function IsClientIdle { local client="${1}" local configFile="${2}" __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local exitCode local configString if [ -f "$configFile" ]; then configString="-c \"$configFile\"" fi Logger "Checking if client [$client] is currently idle." "DEBUG" cmd="$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1" ExecTasks $! "${FUNCNAME[0]}_idle" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING exitCode=$? if [ $exitCode -ne 0 ]; then Logger "Client [$client] is currently backing up." "NOTICE" return $exitCode else return $exitCode fi } function VerifyBackups { local backupDir="${1}" local numberToVerify="${2}" local configFile="${3}" __CheckArguments 2-3 $# "$@" #__WITH_PARANOIA_DEBUG local backupNumber local exitCode local client local configString local interleaveFileArgs=() local interleaveConditionsFileArgs=() local safeExitCodes="0;2;3" # 0: success, 2: warnings, 3: timer conditions not met if [ -f "$configFile" ]; then configString="-c \"$configFile\"" fi for client in "${CLIENT_LIST[@]}"; do # Only backups containing file backup_stats are valid find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | sort -nr | head -n $numberToVerify | sed -e 's/.*\([0-9]\{7\}\).*/\1/' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP" Logger "Can check $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP | wc -l) backups for [$client]." "NOTICE" while IFS=$'\n' read -r backupNumber; do # sed here removes all lines containing only block logs (64 chars + number) # sed regex isn't complete (lacks \+$ because BSD / macOS sed does not like extended regex) Logger "Preparing verification of backup [$backupNumber] for client [$client]." "NOTICE" echo "$BACKUP_EXECUTABLE $configString -C $client -a v -b $backupNumber | sed '/^[rRSDBWfydlLsmnkceaipwxzbMFqYZGOPQEtvVuU]\{64\} [0-9]/d' >> \"$LOG_FILE\" 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP" echo "$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP" done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP" if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP" ]; then interleaveFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP") fi if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP" ]; then interleaveConditionsFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP") fi done InterleaveFiles "${interleaveFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" InterleaveFiles "${interleaveConditionsFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP" Logger "Now running backup verifications, concurrency set to $PARELLEL_VERIFY_CONCURRENCY." "NOTICE" Logger "Executing parallel commands\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG" ExecTasks "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" "${FUNCNAME[0]}" true $SOFT_MAX_EXEC_TIME_PER_VERIFY $HARD_MAX_EXEC_TIME_PER_VERIFY $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING true false false $PARELLEL_VERIFY_CONCURRENCY "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP" $POSTPONE_RETRY $POSTPONE_TIME "$safeExitCodes" exitCode=$? if [ $exitCode -ne 0 ]; then Logger "Client backup verification produced errors [$exitCode]." "ERROR" else Logger "Client backup verification succeed." "NOTICE" fi Logger "Backup verification done." "NOTICE" } function ListOutdatedClients { local backupDir="${1}" local oldDays="${2}" __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local found=false local clientAlertBody local recentBackupTimestamp local CurrentTimestamp Logger "Checking for outdated clients." "NOTICE" for client in "${CLIENT_LIST[@]}"; do if [ -f "$backupDir$client/current/backup_stats" ]; then # This extracts time_end timestamp from last backup stats recentBackupTimestamp=$(awk '/"name": "time_end"/{p=1}{if(p>0){p=p+1}}{if(p==4){print $2}}' "$backupDir$client/current/backup_stats") else recentBackupTimestamp=0 fi # Current timestamp currentTimestamp="$(date +%s)" if [ $((currentTimestamp - (24*3600*oldDays))) -gt $recentBackupTimestamp ]; then Logger "Client [$client] has no backups newer than [$oldDays] days." "ERROR" if [ $CLIENTS_ALERT_OUTDATED == true ]; then clientAlertBody="${CLIENT_ALERT_BODY_OUTDATED/"[CLIENT]"/$client}" clientAlertBody="${clientAlertBody/"[NUMBERDAYS]"/$oldDays}" clientAlertBody="${clientAlertBody/"[INSTANCE]"/$INSTANCE_ID}" if [ "${CLIENT_EMAIL[$client]}" != "" ]; then SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}" Logger "Sent outdated client mail to [${CLIENT_EMAIL[$client]}]." "NOTICE" else Logger "Client [$client] does not have a mail address. Cannot send notification." "ERROR" fi fi found=true fi done if [ $found == false ]; then Logger "No outdated clients found." "NOTICE" else Logger "Outdated client checks done." "NOTICE" fi } function ListQuotaExceedClients { local backupDir="${1}" __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local found=false local lastBackupDir local clientAlertBody local bytesEstimated local quotaExceed local quotaDiff Logger "Checking for clients with exceeded quota." "NOTICE" for client in "${CLIENT_LIST[@]}"; do lastBackupDir=$(find "$backupDir$client" -maxdepth 2 -name "*current") if [ -f "$backupDir$client/current/log.gz" ]; then if zcat "$lastBackupDir/log.gz" | grep quota > /dev/null 2>&1; then bytesEstimated=$(zcat "$lastBackupDir/log.gz" | grep "Bytes estimated") bytesEstimated="${bytesEstimated##*:}" # Remove leading spaces bytesEstimated="${bytesEstimated# *}" quotaExceed=$(zcat "$lastBackupDir/log.gz" | grep "quota") quotaExceed="${quotaExceed##*:}" quotaDiff="$bytesEstimated /$quotaExceed)" Logger "Client [$client] quota exceed ($quotaDiff)." "WARN" if [ $CLIENTS_ALERT_QUOTAS == true ]; then clientAlertBody="${CLIENT_ALERT_BODY_QUOTA/"[CLIENT]"/$client}" clientAlertBody="${clientAlertBody/"[QUOTAEXCEED]"/$quotaDiff}" if [ "${CLIENT_EMAIL[$client]}" != "" ]; then SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}" Logger "Sent quota exceeded mail to [${CLIENT_EMAIL[$client]}]." "NOTICE" else Logger "Client [$client] does not have a mail address. Cannot send notification" "ERROR" fi fi found=true fi else Logger "No valid log file found for analysis of client [$client]." "WARN" fi done if [ $found == false ]; then Logger "No clients with exceeded quota found." "NOTICE" else Logger "Quota checks done." "NOTICE" fi } function VerifyLastWarnings { local backupDir="${1}" __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local found=false Logger "Checking for warnings in last backups." "NOTICE" for client in "${CLIENT_LIST[@]}"; do if [ -f "$backupDir$client/current/log.gz" ]; then if zcat "$backupDir$client/current/log.gz" | grep "WARNING" > /dev/null 2>&1; then Logger "Client [$client] has the following warnings:" "WARN" Logger "$(zcat $backupDir$client/current/log.gz | grep WARNING)" "WARN" found=true fi elif [ -f "$backupDir$client/current/log" ]; then if cat "$backupDir$client/current/log" | grep "WARNING" > /dev/null 2>&1; then Logger "Client [$client] has the following warnings:" "WARN" Logger "$(grep WARNING $backupDir$client/current/log)" "WARN" found=true fi else Logger "No log file found for warning analysis in [$backupDir$client/current]." "WARN" fi done if [ $found == false ]; then Logger "No warnings found in last backups." "NOTICE" fi } function UnstripVSS { local path="${1}" __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG # We need to have a modular temp extension so we will not overwrite potential existing files local tempExtension="$SCRIPT_PID.$TSTAMP.old" if ! type vss_strip > /dev/null 2>&1; then Logger "Could not find vss_strip binary. Please check your path variable." "CRITICAL" exit 1 fi find "$path" -type f -print0 | while IFS= read -r -d $'\0' file; do Logger "Unstripping file [$file]." "NOTICE" mv -f "$file" "$file.$tempExtension" if [ $? -ne 0 ]; then Logger "Could not move [$file] to [$file.$tempExtension] for processing." "WARN" continue else vss_strip -i "$file.$tempExtension" -o "$file" if [ $? -ne 0 ]; then Logger "Could not vss_strip [$file.$tempExtension] to [$file]." "WARN" mv -f "$file.$tempExtension" "$file" if [ $? -ne 0 ]; then Logger "Coult not move back [$file.$tempExtension] to [$file]." "WARN" fi else rm -f "$file.$tempExtension" if [ $? -ne 0 ]; then Logger "Could not delete temporary file [$file.$tempExtension]." "WARN" continue fi fi fi done # Cannot get exitcode since find uses a subshell. Getting exit code from Logger if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then return 2 fi if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then return 1 fi return 0 } function VerifyService { local serviceName="${1}" local serviceType="${2}" __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local serviceNameArray local serviceStatusCommand local serviceStartCommand local i if [ "$serviceName" == "" ]; then Logger "No service name(s) given." "WARN" return fi IFS=',' read -a serviceNameArray <<< "$serviceName" for i in "${serviceNameArray[@]}"; do if [ "$serviceType" == "initv" ]; then serviceStatusCommand="service $i status" serviceStartCommand="service $i start" elif [ "$serviceType" == "systemd" ]; then serviceStatusCommand="systemctl status $i" serviceStartCommand="systemctl start $i" else serviceStatusCommand="service $i status" serviceStartCommand="systemctl start $i" Logger "No valid service type given [$serviceType]. Trying default initV style." "ERROR" fi eval "$serviceStatusCommand" > /dev/null 2>&1 & ExecTasks $! "${FUNCNAME[0]}_statuscmd" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING if [ $? -ne 0 ]; then Logger "Service [$i] is not started. Trying to start it." "WARN" eval "$serviceStartCommand" > /dev/null 2>&1 & ExecTasks $! "${FUNCNAME[0]}_startcmd" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING if [ $? -ne 0 ]; then Logger "Cannot start service [$i]." "CRITICAL" SendAlert false false else Logger "Service [$i] was successfuly started." "WARN" SendAlert false false fi else Logger "Service [$i] is running." "NOTICE" fi done } function DisplayBackupCalendar { local configFile="${1}" __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local client local cmd local backups local months local days local days_expr local cdate local month_count=1 if [ -f "$configFile" ]; then configString="-c \"$configFile\"" fi for client in "${CLIENT_LIST[@]}"; do cmd="$BACKUP_EXECUTABLE $configString -a l -C \"$client\" | grep \"^Backup: \" |awk '{print \$3}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\"" Logger "Running cmd [$cmd]." "DEBUG" eval "$cmd" & ExecTasks $! "${FUNCNAME[0]}" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING if [ $? -ne 0 ]; then Logger "Failed to enumerate backups for client [$client]." "ERROR" elif [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then backups="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" months=$(echo "$backups" | cut -d'-' -f1-2 | sort -V | uniq | tail -n $month_count) Logger "Calendar for client [$client]:" "NOTICE" for month in $months; do days=$(echo "$backups" | grep "$month" | cut -d'-' -f3 | sort | uniq ) days_expr=$(echo "$days" | tr '\n' '|'|rev|cut -c2-|rev) cdate="$(date --date="$month-01" +"%m %Y")" cal $cdate | head -1 cal $cdate | tail -n +2 | sed -r 's/\x5f\x08//g;s/ ([0-9]( |$))/0\1/g' | grep --color -EC10 "((^| )$days_expr( |$))" done else Logger "No backups were found for client [$client]" "NOTICE" fi done } function Init { # Set error exit code if a piped command fails set -o pipefail set -o errtrace trap TrapQuit TERM EXIT HUP QUIT } function Usage { if [ "$IS_STABLE" != true ]; then echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" fi echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" echo "$AUTHOR" echo "$CONTACT" echo "" echo "Usage:" echo "$0 [OPTIONS]" echo "" echo "[OPTIONS]" echo "-d, --backup-dir=\"\" The directory where the client backup directories are" echo "-o, --check-outdated-clients=n Check for clients that don't have backups newer than n days" echo "-v, --verify-last-backups=n Verify the last n backups of all clients" echo "-q, --verify-quotas Check for client quotas" echo "-i, --include-clients=\"\" Comma separated list of clients to include. This list takes grep -e compatible regular expressions, includes prevail excludes" echo "-e, --exclude-clients=\"\" Comma separated list of clients to exclude. This list takes grep -e compatible regular expressions" echo "-c, --config-file=\"\" Path to optional @name@ client configuration file (defaults to /etc/@name@/@name@.conf)" echo "-s, --vss-strip-path=\"\" Run vss_strip for all files in given path" echo "-j, --verify-service=\"\" Comma separated list of @name@ services to check and restart if they aren't running" echo "-w, --verify-warnings Check for warnings in last backup logs" echo "-A, --clients-alert-quotas Use email defined in client config to alert users about exceeded disk quotas (see email template in header)" echo "-a, --clients-alert-outdated Use email defined in client config to alert users about outdated backups (see email template in header)" echo "-z, --clientconfdir=\"\" Path of clientconfdir in order to fetch email addresses when client alerts are used (defaults to /etc/@name@/clientconfdir)" echo "-C, --client=\"\" Specify specific client instead of detecting clients." echo "--calendar Show backup calendar" echo "" echo "Examples:" echo "$0 -d /path/to/@name@/protocol1 -v 3 -c /etc/@name@/@name@.conf" echo "$0 -d /path/to/@name@/protocol2/global/clients --check-outdated-clients7 --exclude-clients=restoreclient,@name@-ui.local" echo "$0 --vss-strip-path=/path/to/restored/files" echo "$0 -j @name@.service" echo "Exclude via regex all clients beginning with 'cli' and otherclient1/2:" echo "$0 --backup-dir=/path/to/@name@/protocol1 --exclude-clients=cli.*,otherclient1,otherclient2" echo "" echo "Additionnal options" echo "--no-maxtime Don't stop checks after the configured maximal time in script" echo "-s, --silent Don't output to stdout, log file only" echo "--errors-only Don't output anything but errors." echo "" echo "--destination-mails=\"\" Space separated list of email adresses where to send warning and error mails" echo "--instance-id=\"\" Arbitrary identifier for log files and alert mails" exit 128 } #### SCRIPT ENTRY POINT DESTINATION_MAILS="" no_maxtime=false ERROR_ALERT=false WARN_ALERT=false CONFIG_FILE="" BACKUP_DIR="" VERIFY_BACKUPS="" INCLUDE_CLIENTS="" EXCLUDE_CLIENTS="" OUTDATED_DAYS="" CLIENT_LIST=() GIVEN_CLIENT="" declare -A CLIENT_EMAIL VSS_STRIP_DIR="" VERIFY_SERVICE=false VERIFY_WARNINGS=false VERIFY_QUOTAS=false CLIENTS_ALERT_QUOTAS=false CLIENTS_ALERT_OUTDATED=false CLIENT_CONF_DIR="" SHOW_BACKUP_CALENDAR=false function GetCommandlineArguments { local isFirstArgument=true if [ $# -eq 0 ] then Usage fi while [ $# -gt 0 ]; do ## Options name is $1, argument is $2 unless there is a separator other than space case $1 in --instance-id=*) INSTANCE_ID="${1##*=}" ;; --silent) _LOGGER_SILENT=true ;; --verbose) _LOGGER_VERBOSE=true ;; --no-maxtime) no_maxtime=true ;; --help|-h|--version) Usage ;; --backup-dir=*) BACKUP_DIR="${1##*=}" ;; -d) BACKUP_DIR="${2}" shift ;; --check-outdated-clients=*) OUTDATED_DAYS="${1##*=}" ;; -o) OUTDATED_DAYS="${2}" shift ;; --verify-last-backups=*) VERIFY_BACKUPS="${1##*=}" ;; -v) VERIFY_BACKUPS="${2}" shift ;; -q|--verify-quotas) VERIFY_QUOTAS=true ;; --include-clients=*) INCLUDE_CLIENTS="${1##*=}" ;; -i) INCLUDE_CLIENTS="${2}" shift ;; --exclude-clients=*) EXCLUDE_CLIENTS="${1##*=}" ;; -e) EXCLUDE_CLIENTS="${2}" shift ;; --config-file=*) CONFIG_FILE="${1##*=}" ;; -c) CONFIG_FILE="${2}" shift ;; -C) GIVEN_CLIENT="${2}" shift ;; --client=*) GIVEN_CLIENT="${1##*=}" ;; --calendar) SHOW_BACKUP_CALENDAR=true ;; --vss-strip-path=*) VSS_STRIP_DIR="${1##*=}" ;; -s) VSS_STRIP_DIR="${2}" shift ;; -j) VERIFY_SERVICE=true VERIFY_SERVICES_NAMES="${2}" shift ;; --verify-service=*) VERIFY_SERVICE=true VERIFY_SERVICE_NAMES="${1##*=}" ;; -A|--client-alert-quotas) CLIENTS_ALERT_QUOTAS=true ;; -a|--client-alert-outdated) CLIENTS_ALERT_OUTDATED=true ;; -z) CLIENT_CONF_DIR="${2}" shift ;; --clientconfdir=*) CLIENT_CONF_DIR="${1##*=}" ;; -w|--verify-warnings) VERIFY_WARNINGS=true ;; --errors-only) _LOGGER_ERR_ONLY=true ;; --destination-mails=*) DESTINATION_MAILS="${1##*=}" ;; --no-maxtime) SOFT_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0 ;; *) if [ $isFirstArgument == false ]; then Logger "Unknown option '${1}'" "CRITICAL" Usage fi ;; esac shift isFirstArgument=false done } GetCommandlineArguments "$@" Init if [ "$LOGFILE" == "" ]; then if [ -w /var/log ]; then LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log" elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log" else LOG_FILE="./$PROGRAM.$INSTANCE_ID.log" fi else LOG_FILE="$LOGFILE" fi if [ ! -w "$(dirname $LOG_FILE)" ]; then echo "Cannot write to log [$(dirname $LOG_FILE)]." else Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" fi DATE=$(date) Logger "---------------------------------------------------------------------" "NOTICE" Logger "$DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS" Logger "---------------------------------------------------------------------" "NOTICE" Logger "Instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" if [ $no_maxtime == true ]; then SOFT_MAX_EXEC_TIME_PER_VERIFY=0 HARD_MAX_EXEC_TIME_PER_VERIFY=0 SOFT_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0 fi if ! type -p "$BACKUP_EXECUTABLE" > /dev/null 2>&1; then Logger "Cannot find [$BACKUP_EXECUTABLE]. Please modify binary path in $0 script header." "CRITICAL" exit 126 fi if [ "$VSS_STRIP_DIR" != "" ]; then if [ -d "$VSS_STRIP_DIR" ]; then UnstripVSS "$VSS_STRIP_DIR" exit $? else Logger "Bogus path given to unstrip [$VSS_STRIP_DIR]." "CRITICAL" exit 1 fi fi if [ "$VERIFY_SERVICE" == true ]; then VerifyService "$VERIFY_SERVICES_NAMES" "$SERVICE_TYPE" fi if [ "$BACKUP_DIR" != "" ]; then if [ ! -d "$BACKUP_DIR" ]; then Logger "Backup dir [$BACKUP_DIR] doesn't exist." "CRITICAL" exit 1 else # Make sure there is only one trailing slash on path BACKUP_DIR="${BACKUP_DIR%/}/" fi fi if [ "$CONFIG_FILE" != "" ]; then if [ ! -f "$CONFIG_FILE" ]; then Logger "Bogus configuration file [$CONFIG_FILE] given." "CRITICAL" exit 1 fi fi if [ "$CLIENT_CONF_DIR" != "" ]; then if [ ! -d "$CLIENT_CONF_DIR" ]; then Logger "Bogus clientconfdir [$CLIENT_CONF_DIR] given." "CRITICAL" exit 1 fi fi ListClients "$BACKUP_DIR" "$CONFIG_FILE" "$CLIENT_CONF_DIR" if [ $SHOW_BACKUP_CALENDAR == true ]; then DisplayBackupCalendar "$CONFIG_FILE" fi if [ $VERIFY_WARNINGS == true ]; then VerifyLastWarnings "$BACKUP_DIR" fi if [ $VERIFY_QUOTAS == true ]; then ListQuotaExceedClients "$BACKUP_DIR" fi if [ "$OUTDATED_DAYS" != "" ]; then if [ $(IsInteger "$OUTDATED_DAYS") -ne 0 ]; then ListOutdatedClients "$BACKUP_DIR" $OUTDATED_DAYS else Logger "Bogus --check-outdated-clients value [$OUTDATED_DAYS]." "CRITICAL" exit 1 fi fi if [ "$VERIFY_BACKUPS" != "" ]; then if [ $(IsInteger "$VERIFY_BACKUPS") -ne 0 ]; then VerifyBackups "$BACKUP_DIR" $VERIFY_BACKUPS "$CONFIG_FILE" else Logger "Bogus --verify-last-backups value [$VERIFY_BACKUPS]." "CRITICAL" exit 1 fi fi # v0.5.0 # - Outdated backups are now chekced against backup_stats file instead backup dir ctime (resolves no outdated backup after copy issue) # - Check against missing service names for service verification # - Ported minor fixes from osync project # - Prettier logs # - Fixed run files cleanup # - Fixed RFC822 mail checks # - Start speedup (changed PoorMansRandomGenerator) # - Fixed potential bash buffer overflow when logging very large file outputs # v0.4.8 # - Removed attachment sending to prevent mailbox clogging # - Fixed possible missing characters from log results (see https://github.com/grke/burp/issues/801 and burp/src/cmd.h) # - Fixed quota warning when no client log file exists # - Smaller fixes ported from ofunctions project # v0.4.6 # - Added command output to logs # - Fixed not using [INSTANCE] placeholder in example text # - Raised default HARD_MAX_EXEC_TIME_PER_VERIFY to 64800 seconds # - Replaced yes/no with true/false booleans # v0.4.4 # - More explicit log messages # - Fix for missing date %N in BSD / MacOS # - Ported some minor fixes from osync project # v0.4.2 # - Added Hakong's backup calendar display (using --calendar) # - Added [INSTANCE] placeholder for email sending # - Client detection now also happens when no data directory is set # - Fixed warning not shown when email address cannot be fetched and client alerts are enabled # - Fixed multiple email adresses not being checked against RFC822 # v0.4.0 first public release, merged into @name@ codebase