From 1db8adfa980356012ba4a4ba5e73c2bcfcae8647 Mon Sep 17 00:00:00 2001 From: deajan Date: Sat, 20 Aug 2016 10:43:22 +0200 Subject: [PATCH 01/16] Reenabled path comparaison --- dev/n_osync.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index f3c7669..9f6e96d 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash #TODO(critical): handle conflict prevalance, especially in sync_attrs function -#TODO(critical): test new WaitForTaskCompletion behavior with self=true PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016081902 +PROGRAM_BUILD=2016082001 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -223,15 +222,16 @@ function CheckReplicaPaths { local pids + # Use direct comparaison before having a portable realpath implementation #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO(verylow): investigate realpath & readlink issues on MSYS and busybox here #TARGET_SYNC_DIR_CANN=$(realpath "${TARGET[1]}) - #if [ "$REMOTE_OPERATION" != "yes" ]; then - # if [ "$INITIATOR_SYNC_DIR_CANN" == "$TARGET_SYNC_DIR_CANN" ]; then - # Logger "Master directory [${INITIATOR[1]}] cannot be the same as target directory." "CRITICAL" - # exit 1 - # fi - #fi + if [ "$REMOTE_OPERATION" != "yes" ]; then + if [ "${INITIATOR[1]}" == "${TARGET[1]}" ]; then + Logger "Initiator and target path [${INITIATOR[1]}] cannot be the same." "CRITICAL" + exit 1 + fi + fi _CheckReplicaPathsLocal "${INITIATOR[1]}" & pids="$!" From fd2dfabce2c0fa9ae55592d54d29efe33a6c880f Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 07:55:03 +0200 Subject: [PATCH 02/16] Made signals posix compliant --- dev/ofunctions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index f4bfd8f..9ba112b 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,6 +1,6 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016081806 +## FUNC_BUILD=2016082201 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -183,7 +183,7 @@ function KillChilds { # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then Logger "Sending SIGTERM to process [$pid]." "DEBUG" - kill -s SIGTERM "$pid" + kill -s TERM "$pid" if [ $? != 0 ]; then sleep 15 Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" From f052422c448a9c6a41dacffbb8b190ce8648ac48 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 08:09:48 +0200 Subject: [PATCH 03/16] Prevent double kill in WaitForTaskCompletion --- dev/ofunctions.sh | 48 ++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 9ba112b..77ffb2c 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -598,30 +598,8 @@ function WaitForTaskCompletion { WAIT_FOR_TASK_COMPLETION="" while [ ${#pidsArray[@]} -gt 0 ]; do + Logger "Hey hey ${#pidsArray[@]} is $(joinString , ${pidsArray[@]})" "WARN" newPidsArray=() - for pid in "${pidsArray[@]}"; do - if kill -0 $pid > /dev/null 2>&1; then - # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) - #TODO(high): have this tested on *BSD, Mac & Win - pidState=$(ps -p$pid -o state= 2 > /dev/null) - if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then - newPidsArray+=($pid) - fi - else - # pid is dead, get it's exit code from wait command - wait $pid - retval=$? - if [ $retval -ne 0 ]; then - errorcount=$((errorcount+1)) - Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" - if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then - WAIT_FOR_TASK_COMPLETION="$pid:$result" - else - WAIT_FOR_TASK_COMPLETION=";$pid:$result" - fi - fi - fi - done Spinner if [ $counting == true ]; then @@ -661,6 +639,30 @@ function WaitForTaskCompletion { fi fi + for pid in "${pidsArray[@]}"; do + if kill -0 $pid > /dev/null 2>&1; then + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + #TODO(high): have this tested on *BSD, Mac & Win + pidState=$(ps -p$pid -o state= 2 > /dev/null) + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + newPidsArray+=($pid) + fi + else + # pid is dead, get it's exit code from wait command + wait $pid + retval=$? + if [ $retval -ne 0 ]; then + errorcount=$((errorcount+1)) + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then + WAIT_FOR_TASK_COMPLETION="$pid:$result" + else + WAIT_FOR_TASK_COMPLETION=";$pid:$result" + fi + fi + fi + done + pidsArray=("${newPidsArray[@]}") sleep $SLEEP_TIME done From df2cfe8fa556739de61b0751bd030a4f553f7888 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 08:12:08 +0200 Subject: [PATCH 04/16] Removed unused code --- dev/ofunctions.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 77ffb2c..92e9069 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,6 +1,6 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016082201 +## FUNC_BUILD=2016082202 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -580,7 +580,6 @@ function WaitForTaskCompletion { Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once local log_ttime=0 # local time instance for comparaison local seconds_begin=$SECONDS # Seconds since the beginning of the script @@ -620,7 +619,6 @@ function WaitForTaskCompletion { if [ $exec_time -gt $soft_max_time ]; then if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" - soft_alert=1 SendAlert fi From bcbd43fafb953befd00da95d4ed6680e07c31263 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 08:31:48 +0200 Subject: [PATCH 05/16] Fixed code removal --- dev/ofunctions.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 92e9069..1c467b0 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,6 +1,6 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016082202 +## FUNC_BUILD=2016082204 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -580,6 +580,7 @@ function WaitForTaskCompletion { Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once local log_ttime=0 # local time instance for comparaison local seconds_begin=$SECONDS # Seconds since the beginning of the script @@ -597,7 +598,6 @@ function WaitForTaskCompletion { WAIT_FOR_TASK_COMPLETION="" while [ ${#pidsArray[@]} -gt 0 ]; do - Logger "Hey hey ${#pidsArray[@]} is $(joinString , ${pidsArray[@]})" "WARN" newPidsArray=() Spinner @@ -619,6 +619,7 @@ function WaitForTaskCompletion { if [ $exec_time -gt $soft_max_time ]; then if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + soft_alert=1 SendAlert fi From ff6a0b95fcd966cbea1619754bc74789d278715c Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 08:32:24 +0200 Subject: [PATCH 06/16] Fixed RunAfterHook not executed on normal runs --- dev/n_osync.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 9f6e96d..13358f6 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash #TODO(critical): handle conflict prevalance, especially in sync_attrs function +#TODO(critical): writelockfiles remote does not shut execution PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" @@ -98,9 +99,7 @@ function TrapQuit { exitcode=2 # Warning exit code must not force daemon mode to quit else UnlockReplicas - if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then - RunAfterHook - fi + RunAfterHook CleanUp Logger "$PROGRAM finished." "NOTICE" exitcode=0 @@ -1569,10 +1568,10 @@ function Init { # Do not use exit and quit traps if osync runs in monitor mode if [ $sync_on_changes -eq 0 ]; then - trap TrapStop SIGINT SIGHUP SIGTERM SIGQUIT + trap TrapStop INT HUP TERM QUIT trap TrapQuit EXIT else - trap TrapQuit SIGTERM EXIT SIGHUP SIGQUIT + trap TrapQuit TERM EXIT HUP QUIT fi local uri From e1bcaea6a392fc91509528655dd256c435412250 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 09:07:58 +0200 Subject: [PATCH 07/16] Updated serials --- dev/n_osync.sh | 2 +- dev/ofunctions.sh.old | 1310 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1311 insertions(+), 1 deletion(-) create mode 100644 dev/ofunctions.sh.old diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 13358f6..a982ac4 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016082001 +PROGRAM_BUILD=2016082201 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG diff --git a/dev/ofunctions.sh.old b/dev/ofunctions.sh.old new file mode 100644 index 0000000..9707c71 --- /dev/null +++ b/dev/ofunctions.sh.old @@ -0,0 +1,1310 @@ +#### MINIMAL-FUNCTION-SET BEGIN #### + +## FUNC_BUILD=2016081501 +## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode +if ! type "$BASH" > /dev/null; then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 +fi + +## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. +KEEP_LOGGING=1801 + +## Correct output of sort command (language agnostic sorting) +export LC_ALL=C + +# Standard alert mail body +MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." + +# Environment variables that can be overriden by programs +_DRYRUN=0 +_SILENT=0 +_LOGGER_PREFIX="date" +_LOGGER_STDERR=0 + + +# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags +ERROR_ALERT=0 +WARN_ALERT=0 + +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG + _DEBUG=yes #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG + +## allow debugging from command line with _DEBUG=yes +if [ ! "$_DEBUG" == "yes" ]; then + _DEBUG=no + SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys + _VERBOSE=0 +else + SLEEP_TIME=1 + trap 'TrapError ${LINENO} $?' ERR + _VERBOSE=1 +fi + +SCRIPT_PID=$$ + +LOCAL_USER=$(whoami) +LOCAL_HOST=$(hostname) + +## Default log file until config file is loaded +if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.log" +elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then + LOG_FILE="$HOME/$PROGRAM.log" +else + LOG_FILE="./$PROGRAM.log" +fi + +## Default directory where to store temporary run files +if [ -w /tmp ]; then + RUN_DIR=/tmp +elif [ -w /var/tmp ]; then + RUN_DIR=/var/tmp +else + RUN_DIR=. +fi + + +# Default alert attachment filename +ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" + +# Set error exit code if a piped command fails + set -o pipefail + set -o errtrace + + +function Dummy { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + sleep $SLEEP_TIME +} + +# Sub function of Logger +function _Logger { + local svalue="${1}" # What to log to stdout + local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value + local evalue="${3}" # What to log to stderr + + echo -e "$lvalue" >> "$LOG_FILE" + + if [ "$_LOGGER_STDERR" -eq 1 ]; then + cat <<< "$evalue" 1>&2 + elif [ "$_SILENT" -eq 0 ]; then + echo -e "$svalue" + fi +} + +# General log function with log levels +function Logger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then + prefix="$(date) - " + else + prefix="" + fi + + if [ "$level" == "CRITICAL" ]; then + _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" + ERROR_ALERT=1 + return + elif [ "$level" == "ERROR" ]; then + _Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" + ERROR_ALERT=1 + return + elif [ "$level" == "WARN" ]; then + _Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value" + WARN_ALERT=1 + return + elif [ "$level" == "NOTICE" ]; then + _Logger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "$prefix$value" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _Logger "\e[41mLogger function called without proper loglevel.\e[0m" + _Logger "$prefix$value" + fi +} + +# QuickLogger subfunction, can be called directly +function _QuickLogger { + local value="${1}" + local destination="${2}" # Destination: stdout, log, both + + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then + echo -e "$(date) - $value" >> "$LOG_FILE" + elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then + echo -e "$value" + fi +} + +# Generic quick logging function +function QuickLogger { + local value="${1}" + + __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_SILENT" -eq 1 ]; then + _QuickLogger "$value" "log" + else + _QuickLogger "$value" "stdout" + fi +} + +# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X +function KillChilds { + local pid="${1}" # Parent pid to kill + local self="${2:-false}" + + + if children="$(pgrep -P "$pid")"; then + for child in $children; do + Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG + KillChilds "$child" true + done + fi + # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing + if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then + Logger "Sending SIGTERM to process [$pid]." "DEBUG" + kill -s SIGTERM "$pid" + if [ $? != 0 ]; then + sleep 15 + Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" + kill -9 "$pid" + if [ $? != 0 ]; then + Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" + return 1 + fi + else + return 0 + fi + else + return 0 + fi +} + +function KillAllChilds { + local pids="${1}" # List of parent pids to kill separated by semi-colon + + __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local errorcount=0 + + IFS=';' read -a pidsArray <<< "$pids" + for pid in "${pidsArray[@]}"; do + KillChilds $pid + if [ $? != 0 ]; then + errorcount=$((errorcount+1)) + fi + done + return $errorcount +} + +# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending +function SendAlert { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local mail_no_attachment= + local attachment_command= + local subject= + + # Windows specific settings + local encryption_string= + local auth_string= + + if [ "$DESTINATION_MAILS" == "" ]; then + return 0 + fi + + if [ "$_DEBUG" == "yes" ]; then + Logger "Debug mode, no warning mail will be sent." "NOTICE" + return 0 + fi + + # + if [ "$_QUICK_SYNC" == "2" ]; then + Logger "Current task is a quicksync task. Will not send any alert." "NOTICE" + return 0 + fi + # + + eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" + if [ $? != 0 ]; then + Logger "Cannot create [$ALERT_LOG_FILE]" "WARN" + mail_no_attachment=1 + else + mail_no_attachment=0 + fi + MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE") + if [ $ERROR_ALERT -eq 1 ]; then + subject="Error alert for $INSTANCE_ID" + elif [ $WARN_ALERT -eq 1 ]; then + subject="Warning alert for $INSTANCE_ID" + else + subject="Alert for $INSTANCE_ID" + fi + + if [ "$mail_no_attachment" -eq 0 ]; then + attachment_command="-a $ALERT_LOG_FILE" + fi + if type mutt > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command + if [ $? != 0 ]; then + Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN" + else + Logger "Sent alert mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then + attachment_command="-A $ALERT_LOG_FILE" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then + attachment_command="-a$ALERT_LOG_FILE" + else + attachment_command="" + fi + echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN" + echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent alert mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent alert mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent alert mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + # Windows specific + if type "mailsend.exe" > /dev/null 2>&1 ; then + + if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then + Logger "Bogus smtp encryption, assuming none." "WARN" + encryption_string= + elif [ "$SMTP_ENCRYPTION" == "tls" ]; then + encryption_string=-starttls + elif [ "$SMTP_ENCRYPTION" == "ssl" ]:; then + encryption_string=-ssl + fi + if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then + auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\"" + fi + $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string + if [ $? != 0 ]; then + Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" + else + Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" + return 0 + fi + fi + + # Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/) + if type sendemail > /dev/null 2>&1 ; then + if [ "$SMTP_USER" != "" ] && [ "$SMTP_PASSWORD" != "" ]; then + SMTP_OPTIONS="-xu $SMTP_USER -xp $SMTP_PASSWORD" + else + SMTP_OPTIONS="" + fi + $(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 + if [ $? != 0 ]; then + Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN" + else + Logger "Sent alert mail using sendemail command without attachment." "NOTICE" + return 0 + fi + fi + + # pfSense specific + if [ -f /usr/local/bin/mail.php ]; then + echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject" + if [ $? != 0 ]; then + Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" + else + Logger "Sent alert mail using pfSense mail.php." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it is critical that no alert can be sent + Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue + + # Delete tmp log file + if [ -f "$ALERT_LOG_FILE" ]; then + rm "$ALERT_LOG_FILE" + fi +} + +# Generic email sending function. +# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or "" +# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" +# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend) +# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or "" +# smtp_server.domain.tld is mandatory, as is smtp_port (should be 25, 465 or 587) +# encryption can be set to tls, ssl or none +# smtp_user and smtp_password are optional +# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "sender_email@example.com" "smtp_server.domain.tld" "smtp_port" "encryption" "smtp_user" "smtp_password" +function SendEmail { + local subject="${1}" + local message="${2}" + local destination_mails="${3}" + local attachment="${4}" + local sender_email="${5}" + local smtp_server="${6}" + local smtp_port="${7}" + local encryption="${8}" + local smtp_user="${9}" + local smtp_password="${10}" + + # CheckArguments will report a warning that can be ignored if used in Windows with paranoia debug enabled + __CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local mail_no_attachment= + local attachment_command= + + local encryption_string= + local auth_string= + + if [ ! -f "$attachment" ]; then + attachment_command="-a $ALERT_LOG_FILE" + mail_no_attachment=1 + else + mail_no_attachment=0 + fi + + if type mutt > /dev/null 2>&1 ; then + echo "$message" | $(type -p mutt) -x -s "$subject" "$destination_mails" $attachment_command + if [ $? != 0 ]; then + Logger "Cannot send mail via $(type -p mutt) !!!" "WARN" + else + Logger "Sent mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then + attachment_command="-A $attachment" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then + attachment_command="-a$attachment" + else + attachment_command="" + fi + echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destination_mails" + if [ $? != 0 ]; then + Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" + echo "$message" | $(type -p mail) -s "$subject" "$destination_mails" + if [ $? != 0 ]; then + Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destination_mails" + if [ $? != 0 ]; then + Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + # Windows specific + if type "mailsend.exe" > /dev/null 2>&1 ; then + if [ "$sender_email" == "" ]; then + Logger "Missing sender email." "ERROR" + return 1 + fi + if [ "$smtp_server" == "" ]; then + Logger "Missing smtp port." "ERROR" + return 1 + fi + if [ "$smtp_port" == "" ]; then + Logger "Missing smtp port, assuming 25." "WARN" + smtp_port=25 + fi + if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then + Logger "Bogus smtp encryption, assuming none." "WARN" + encryption_string= + elif [ "$encryption" == "tls" ]; then + encryption_string=-starttls + elif [ "$encryption" == "ssl" ]:; then + encryption_string=-ssl + fi + if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then + auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\"" + fi + $(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string + if [ $? != 0 ]; then + Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" + else + Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" + return 0 + fi + fi + + # pfSense specific + if [ -f /usr/local/bin/mail.php ]; then + echo "$message" | /usr/local/bin/mail.php -s="$subject" + if [ $? != 0 ]; then + Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" + else + Logger "Sent mail using pfSense mail.php." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it is critical that no alert can be sent + Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue +} + +function TrapError { + local job="$0" + local line="$1" + local code="${2:-1}" + if [ $_SILENT -eq 0 ]; then + echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" + fi +} + +function LoadConfigFile { + local config_file="${1}" + __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + + if [ ! -f "$config_file" ]; then + Logger "Cannot load configuration file [$config_file]. Cannot start." "CRITICAL" + exit 1 + elif [[ "$1" != *".conf" ]]; then + Logger "Wrong configuration file supplied [$config_file]. Cannot start." "CRITICAL" + exit 1 + else + grep '^[^ ]*=[^;&]*' "$config_file" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" # WITHOUT COMMENTS + # Shellcheck source=./sync.conf + source "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + fi + + CONFIG_FILE="$config_file" +} + +function Spinner { + if [ $_SILENT -eq 1 ]; then + return 0 + fi + + case $toggle + in + 1) + echo -n " \ " + echo -ne "\r" + toggle="2" + ;; + + 2) + echo -n " | " + echo -ne "\r" + toggle="3" + ;; + + 3) + echo -n " / " + echo -ne "\r" + toggle="4" + ;; + + *) + echo -n " - " + echo -ne "\r" + toggle="1" + ;; + esac +} + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + +# Time control function for background processes, suitable for multiple synchronous processes +# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 +# Warning: Don't imbricate this function into another run if you plan to use the global variable output +function WaitForTaskCompletion { + local pids="${1}" # pids to wait for, separated by semi-colon + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + local exit_on_error="${5:-false}" # Should the function exit on subprocess errors + local counting="${6:-true}" # Count time since function launch if true, script launch if false + + Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + local errorcount=0 # Number of pids that finished with errors + + local pidCount # number of given pids + local pidState # State of the process + + IFS=';' read -a pidsArray <<< "$pids" + pidCount=${#pidsArray[@]} + + WAIT_FOR_TASK_COMPLETION="" + + #TODO: need to find a way to properly handle processes in unterruptible sleep state + + while [ ${#pidsArray[@]} -gt 0 ]; do + newPidsArray=() + for pid in "${pidsArray[@]}"; do + if kill -0 $pid > /dev/null 2>&1; then + # Handle uninterruptible sleep state or zombies by ommiting them from running process array + #TODO(high): have this tested on *BSD, Mac & Win + pidState=$(ps -p$pid -o state=) + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + newPidsArray+=($pid) + fi + else + wait $pid + result=$? + if [ $result -ne 0 ]; then + errorcount=$((errorcount+1)) + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then + WAIT_FOR_TASK_COMPLETION="$pid:$result" + else + WAIT_FOR_TASK_COMPLETION=";$pid:$result" + fi + fi + fi + done + + Spinner + if [ $counting == true ]; then + exec_time=$(($SECONDS - $seconds_begin)) + else + exec_time=$SECONDS + fi + + if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then + log_ttime=$exec_time + Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + fi + + if [ $exec_time -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then + Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + soft_alert=1 + SendAlert + + fi + if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then + Logger "Max hard execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" + KillChilds $pid + if [ $? == 0 ]; then + Logger "Task stopped successfully." "NOTICE" + else + Logger "Could not stop task." "ERROR" + fi + SendAlert + errrorcount=$((errorcount+1)) + fi + fi + + pidsArray=("${newPidsArray[@]}") + sleep $SLEEP_TIME + done + + Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then + Logger "Stopping execution." "CRITICAL" + exit 1337 + else + return $errorcount + fi +} + +function CleanUp { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_DEBUG" != "yes" ]; then + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID" + # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.tmp" + fi +} + +#### MINIMAL-FUNCTION-SET END #### + +# obsolete, use StripQuotes +function SedStripQuotes { + echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") +} + +# Usage: var=$(StripSingleQuotes "$var") +function StripSingleQuotes { + local string="${1}" + string="${string/#\'/}" # Remove singlequote if it begins string + string="${string/%\'/}" # Remove singlequote if it ends string + echo "$string" +} + +# Usage: var=$(StripDoubleQuotes "$var") +function StripDoubleQuotes { + local string="${1}" + string="${string/#\"/}" + string="${string/%\"/}" + echo "$string" +} + +function StripQuotes { + local string="${1}" + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" +} + +function EscapeSpaces { + local string="${1}" # String on which spaces will be escaped + echo "${string// /\ }" +} + +function IsNumeric { + eval "local value=\"${1}\"" # Needed so variable variables can be processed + + local re="^-?[0-9]+([.][0-9]+)?$" + if [[ $value =~ $re ]]; then + echo 1 + else + echo 0 + fi +} + +## from https://gist.github.com/cdown/1163649 +function urlEncode { + local length="${#1}" + + local LANG=C + for (( i = 0; i < length; i++ )); do + local c="${1:i:1}" + case $c in + [a-zA-Z0-9.~_-]) + printf "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} + +function urlDecode { + local url_encoded="${1//+/ }" + + printf '%b' "${url_encoded//%/\\x}" +} + +function GetLocalOS { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local local_os_var= + + local_os_var="$(uname -spio 2>&1)" + if [ $? != 0 ]; then + local_os_var="$(uname -v 2>&1)" + if [ $? != 0 ]; then + local_os_var="$(uname)" + fi + fi + + case $local_os_var in + *"Linux"*) + LOCAL_OS="Linux" + ;; + *"BSD"*) + LOCAL_OS="BSD" + ;; + *"MINGW32"*|*"CYGWIN"*) + LOCAL_OS="msys" + ;; + *"Darwin"*) + LOCAL_OS="MacOSX" + ;; + *) + if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option + Logger "Running on unknown local OS [$local_os_var]." "WARN" + return + fi + Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" + exit 1 + ;; + esac + Logger "Local OS: [$local_os_var]." "DEBUG" +} + +function GetRemoteOS { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local cmd= + local remote_os_var= + + + if [ "$REMOTE_OPERATION" == "yes" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-1" false true + retval=$? + if [ $retval != 0 ]; then + cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-2" false true + retval=$? + if [ $retval != 0 ]; then + cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-3" false true + retval=$? + if [ $retval != 0 ]; then + Logger "Cannot Get remote OS type." "ERROR" + fi + fi + fi + + remote_os_var=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") + + case $remote_os_var in + *"Linux"*) + REMOTE_OS="Linux" + ;; + *"BSD"*) + REMOTE_OS="BSD" + ;; + *"MINGW32"*|*"CYGWIN"*) + REMOTE_OS="msys" + ;; + *"Darwin"*) + REMOTE_OS="MacOSX" + ;; + *"ssh"*|*"SSH"*) + Logger "Cannot connect to remote system." "CRITICAL" + exit 1 + ;; + *) + if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option + Logger "Running on unknown remote OS [$remote_os_var]." "WARN" + return + fi + Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" + Logger "Remote OS said:\n$remote_os_var" "CRITICAL" + exit 1 + esac + + Logger "Remote OS: [$remote_os_var]." "DEBUG" + fi +} + +function RunLocalCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 0 + fi + + Logger "Running command [$command] on local host." "NOTICE" + eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & + WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} false true + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +## Runs remote command $1 and waits for completition in $2 seconds +function RunRemoteCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 0 + fi + + Logger "Running command [$command] on remote host." "NOTICE" + cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} false true + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]) + then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +function RunBeforeHook { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local pids= + + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$!" + fi + + if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false true + fi +} + +function RunAfterHook { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local pids + + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$!" + fi + + if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false true + fi +} + +function CheckConnectivityRemoteHost { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + + if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then + eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & + WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} false true + if [ $? != 0 ]; then + Logger "Cannot ping $REMOTE_HOST" "ERROR" + return 1 + fi + fi + fi +} + +function CheckConnectivity3rdPartyHosts { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local remote_3rd_party_success + local pids + + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + + if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then + remote_3rd_party_success=0 + for i in $REMOTE_3RD_PARTY_HOSTS + do + eval "$PING_CMD $i > /dev/null 2>&1" & + WaitForTaskCompletion $! 10 360 ${FUNCNAME[0]} false true + if [ $? != 0 ]; then + Logger "Cannot ping 3rd party host $i" "NOTICE" + else + remote_3rd_party_success=1 + fi + done + + if [ $remote_3rd_party_success -ne 1 ]; then + Logger "No remote 3rd party host responded to ping. No internet ?" "ERROR" + return 1 + else + return 0 + fi + fi + fi +} + +#__BEGIN_WITH_PARANOIA_DEBUG +function __CheckArguments { + # Checks the number of arguments of a function and raises an error if some are missing + + if [ "$_DEBUG" == "yes" ]; then + local number_of_arguments="${1}" # Number of arguments the tested function should have + local number_of_given_arguments="${2}" # Number of arguments that have been passed + local function_name="${3}" # Function name that called __CheckArguments + + if [ "$_PARANOIA_DEBUG" == "yes" ]; then + Logger "Entering function [$function_name]." "DEBUG" + fi + + # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) + # If any of the arguments contains spaces, bash things there are two aguments + # In order to avoid this, we need to iterate over ${4} and count + + local iterate=4 + local fetch_arguments=1 + local arg_list="" + while [ $fetch_arguments -eq 1 ]; do + cmd='argument=${'$iterate'}' + eval $cmd + if [ "$argument" = "" ]; then + fetch_arguments=0 + else + arg_list="$arg_list [Argument $(($iterate-3)): $argument]" + iterate=$(($iterate+1)) + fi + done + local counted_arguments=$((iterate-4)) + + if [ $counted_arguments -ne $number_of_arguments ]; then + Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR" + Logger "Arguments passed: $arg_list" "ERROR" + fi + fi +} + +#__END_WITH_PARANOIA_DEBUG + +function RsyncPatternsAdd { + local pattern_type="${1}" # exclude or include + local pattern="${2}" + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local rest + + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$pattern" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str=${rest%%;*} + # Handle the last case + if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest=${rest#*$PATH_SEPARATOR_CHAR} + fi + if [ "$RSYNC_PATTERNS" == "" ]; then + RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" + else + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" + fi + done + set +f +} + +function RsyncPatternsFromAdd { + local pattern_type="${1}" + local pattern_from="${2}" + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $pattern_from)" == "$pattern_from" ]; then + pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" + fi + + if [ -e "$pattern_from" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" + fi +} + +function RsyncPatterns { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then + if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" + fi + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" + fi + if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + fi + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" + fi + elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then + if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" + fi + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" + fi + if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" + fi + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" + fi + else + Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" + fi +} + +function PreInit { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + ## SSH compression + if [ "$SSH_COMPRESSION" != "no" ]; then + SSH_COMP=-C + else + SSH_COMP= + fi + + ## Ignore SSH known host verification + if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then + SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + fi + + ## Support for older config files without RSYNC_EXECUTABLE option + if [ "$RSYNC_EXECUTABLE" == "" ]; then + RSYNC_EXECUTABLE=rsync + fi + + ## Sudo execution option + if [ "$SUDO_EXEC" == "yes" ]; then + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="sudo $RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="sudo" + else + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="$RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="" + fi + + ## Set rsync default arguments + RSYNC_ARGS="-rltD" + if [ "$_DRYRUN" -eq 1 ]; then + RSYNC_DRY_ARG="-n" + DRY_WARNING="/!\ DRY RUN" + else + RSYNC_DRY_ARG="" + fi + + RSYNC_ATTR_ARGS="" + if [ "$PRESERVE_PERMISSIONS" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" + fi + if [ "$PRESERVE_OWNER" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" + fi + if [ "$PRESERVE_GROUP" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" + fi + if [ "$PRESERVE_ACL" == "yes" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" + fi + if [ "$PRESERVE_XATTR" == "yes" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" + fi + if [ "$RSYNC_COMPRESS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -z" + fi + if [ "$COPY_SYMLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -K" + fi + if [ "$PRESERVE_HARDLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -H" + fi + if [ "$CHECKSUM" == "yes" ]; then + RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + fi + + if [ "$PARTIAL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELTA_COPIES" != "no" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + else + RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + fi + + ## Set compression executable and extension + COMPRESSION_LEVEL=3 + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + +function PostInit { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + # Define remote commands + SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" +} + +function InitLocalOSSettings { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + ## If running under Msys, some commands do not run the same way + ## Using mingw version of find instead of windows one + ## Getting running processes is quite different + ## Ping command is not the same + if [ "$LOCAL_OS" == "msys" ]; then + FIND_CMD=$(dirname $BASH)/find + PING_CMD='$SYSTEMROOT\system32\ping -n 2' + else + FIND_CMD=find + PING_CMD="ping -c 2 -i .2" + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then + STAT_CMD="stat -f \"%Sm\"" + STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" + else + STAT_CMD="stat --format %y" + STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" + fi +} + +function InitRemoteOSSettings { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + #TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes + ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) + if [ "$PRESERVE_EXECUTABILITY" != "no" ];then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" + fi + fi + + if [ "$REMOTE_OS" == "msys" ]; then + REMOTE_FIND_CMD=$(dirname $BASH)/find + else + REMOTE_FIND_CMD=find + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then + REMOTE_STAT_CMD="stat -f \"%Sm\"" + REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" + else + REMOTE_STAT_CMD="stat --format %y" + REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" + fi + +} + +## END Generic functions From cf89a2fb31e1da460632d5480963620c82e3d664 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 09:09:59 +0200 Subject: [PATCH 08/16] Rebuilt targets --- dev/debug_osync.sh | 78 +-- dev/ofunctions.sh.old | 1310 ----------------------------------------- osync.sh | 78 +-- 3 files changed, 78 insertions(+), 1388 deletions(-) delete mode 100644 dev/ofunctions.sh.old diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index cdb9e71..6d0c483 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash #TODO(critical): handle conflict prevalance, especially in sync_attrs function -#TODO(critical): test new WaitForTaskCompletion behavior with self=true +#TODO(critical): writelockfiles remote does not shut execution PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016081902 +PROGRAM_BUILD=2016082201 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -47,7 +47,7 @@ IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016081805 +## FUNC_BUILD=2016082204 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -230,7 +230,7 @@ function KillChilds { # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then Logger "Sending SIGTERM to process [$pid]." "DEBUG" - kill -s SIGTERM "$pid" + kill -s TERM "$pid" if [ $? != 0 ]; then sleep 15 Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" @@ -646,29 +646,6 @@ function WaitForTaskCompletion { while [ ${#pidsArray[@]} -gt 0 ]; do newPidsArray=() - for pid in "${pidsArray[@]}"; do - if kill -0 $pid > /dev/null 2>&1; then - # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) - #TODO(high): have this tested on *BSD, Mac & Win - pidState=$(ps -p$pid -o state= 2 > /dev/null) - if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then - newPidsArray+=($pid) - fi - else - # pid is dead, get it's exit code from wait command - wait $pid - retval=$? - if [ $retval -ne 0 ]; then - errorcount=$((errorcount+1)) - Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" - if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then - WAIT_FOR_TASK_COMPLETION="$pid:$result" - else - WAIT_FOR_TASK_COMPLETION=";$pid:$result" - fi - fi - fi - done Spinner if [ $counting == true ]; then @@ -708,6 +685,30 @@ function WaitForTaskCompletion { fi fi + for pid in "${pidsArray[@]}"; do + if kill -0 $pid > /dev/null 2>&1; then + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + #TODO(high): have this tested on *BSD, Mac & Win + pidState=$(ps -p$pid -o state= 2 > /dev/null) + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + newPidsArray+=($pid) + fi + else + # pid is dead, get it's exit code from wait command + wait $pid + retval=$? + if [ $retval -ne 0 ]; then + errorcount=$((errorcount+1)) + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then + WAIT_FOR_TASK_COMPLETION="$pid:$result" + else + WAIT_FOR_TASK_COMPLETION=";$pid:$result" + fi + fi + fi + done + pidsArray=("${newPidsArray[@]}") sleep $SLEEP_TIME done @@ -1037,7 +1038,7 @@ function CheckConnectivity3rdPartyHosts { for i in $REMOTE_3RD_PARTY_HOSTS do eval "$PING_CMD $i > /dev/null 2>&1" & - WaitForTaskCompletion $! 10 360 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot ping 3rd party host $i" "NOTICE" else @@ -1417,9 +1418,7 @@ function TrapQuit { exitcode=2 # Warning exit code must not force daemon mode to quit else UnlockReplicas - if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then - RunAfterHook - fi + RunAfterHook CleanUp Logger "$PROGRAM finished." "NOTICE" exitcode=0 @@ -1541,15 +1540,16 @@ function CheckReplicaPaths { local pids + # Use direct comparaison before having a portable realpath implementation #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO(verylow): investigate realpath & readlink issues on MSYS and busybox here #TARGET_SYNC_DIR_CANN=$(realpath "${TARGET[1]}) - #if [ "$REMOTE_OPERATION" != "yes" ]; then - # if [ "$INITIATOR_SYNC_DIR_CANN" == "$TARGET_SYNC_DIR_CANN" ]; then - # Logger "Master directory [${INITIATOR[1]}] cannot be the same as target directory." "CRITICAL" - # exit 1 - # fi - #fi + if [ "$REMOTE_OPERATION" != "yes" ]; then + if [ "${INITIATOR[1]}" == "${TARGET[1]}" ]; then + Logger "Initiator and target path [${INITIATOR[1]}] cannot be the same." "CRITICAL" + exit 1 + fi + fi _CheckReplicaPathsLocal "${INITIATOR[1]}" & pids="$!" @@ -2887,10 +2887,10 @@ function Init { # Do not use exit and quit traps if osync runs in monitor mode if [ $sync_on_changes -eq 0 ]; then - trap TrapStop SIGINT SIGHUP SIGTERM SIGQUIT + trap TrapStop INT HUP TERM QUIT trap TrapQuit EXIT else - trap TrapQuit SIGTERM EXIT SIGHUP SIGQUIT + trap TrapQuit TERM EXIT HUP QUIT fi local uri diff --git a/dev/ofunctions.sh.old b/dev/ofunctions.sh.old deleted file mode 100644 index 9707c71..0000000 --- a/dev/ofunctions.sh.old +++ /dev/null @@ -1,1310 +0,0 @@ -#### MINIMAL-FUNCTION-SET BEGIN #### - -## FUNC_BUILD=2016081501 -## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr - -## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode -if ! type "$BASH" > /dev/null; then - echo "Please run this script only with bash shell. Tested on bash >= 3.2" - exit 127 -fi - -## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. -KEEP_LOGGING=1801 - -## Correct output of sort command (language agnostic sorting) -export LC_ALL=C - -# Standard alert mail body -MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." - -# Environment variables that can be overriden by programs -_DRYRUN=0 -_SILENT=0 -_LOGGER_PREFIX="date" -_LOGGER_STDERR=0 - - -# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags -ERROR_ALERT=0 -WARN_ALERT=0 - -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG - _DEBUG=yes #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG - -## allow debugging from command line with _DEBUG=yes -if [ ! "$_DEBUG" == "yes" ]; then - _DEBUG=no - SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys - _VERBOSE=0 -else - SLEEP_TIME=1 - trap 'TrapError ${LINENO} $?' ERR - _VERBOSE=1 -fi - -SCRIPT_PID=$$ - -LOCAL_USER=$(whoami) -LOCAL_HOST=$(hostname) - -## Default log file until config file is loaded -if [ -w /var/log ]; then - LOG_FILE="/var/log/$PROGRAM.log" -elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then - LOG_FILE="$HOME/$PROGRAM.log" -else - LOG_FILE="./$PROGRAM.log" -fi - -## Default directory where to store temporary run files -if [ -w /tmp ]; then - RUN_DIR=/tmp -elif [ -w /var/tmp ]; then - RUN_DIR=/var/tmp -else - RUN_DIR=. -fi - - -# Default alert attachment filename -ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" - -# Set error exit code if a piped command fails - set -o pipefail - set -o errtrace - - -function Dummy { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - sleep $SLEEP_TIME -} - -# Sub function of Logger -function _Logger { - local svalue="${1}" # What to log to stdout - local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value - local evalue="${3}" # What to log to stderr - - echo -e "$lvalue" >> "$LOG_FILE" - - if [ "$_LOGGER_STDERR" -eq 1 ]; then - cat <<< "$evalue" 1>&2 - elif [ "$_SILENT" -eq 0 ]; then - echo -e "$svalue" - fi -} - -# General log function with log levels -function Logger { - local value="${1}" # Sentence to log (in double quotes) - local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL - - if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="TIME: $SECONDS - " - elif [ "$_LOGGER_PREFIX" == "date" ]; then - prefix="$(date) - " - else - prefix="" - fi - - if [ "$level" == "CRITICAL" ]; then - _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" - ERROR_ALERT=1 - return - elif [ "$level" == "ERROR" ]; then - _Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" - ERROR_ALERT=1 - return - elif [ "$level" == "WARN" ]; then - _Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value" - WARN_ALERT=1 - return - elif [ "$level" == "NOTICE" ]; then - _Logger "$prefix$value" - return - elif [ "$level" == "DEBUG" ]; then - if [ "$_DEBUG" == "yes" ]; then - _Logger "$prefix$value" - return - fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG - _Logger "$prefix$value" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG - else - _Logger "\e[41mLogger function called without proper loglevel.\e[0m" - _Logger "$prefix$value" - fi -} - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - if [ "$_SILENT" -eq 1 ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} - -# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X -function KillChilds { - local pid="${1}" # Parent pid to kill - local self="${2:-false}" - - - if children="$(pgrep -P "$pid")"; then - for child in $children; do - Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG - KillChilds "$child" true - done - fi - # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing - if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then - Logger "Sending SIGTERM to process [$pid]." "DEBUG" - kill -s SIGTERM "$pid" - if [ $? != 0 ]; then - sleep 15 - Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" - kill -9 "$pid" - if [ $? != 0 ]; then - Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" - return 1 - fi - else - return 0 - fi - else - return 0 - fi -} - -function KillAllChilds { - local pids="${1}" # List of parent pids to kill separated by semi-colon - - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local errorcount=0 - - IFS=';' read -a pidsArray <<< "$pids" - for pid in "${pidsArray[@]}"; do - KillChilds $pid - if [ $? != 0 ]; then - errorcount=$((errorcount+1)) - fi - done - return $errorcount -} - -# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending -function SendAlert { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local mail_no_attachment= - local attachment_command= - local subject= - - # Windows specific settings - local encryption_string= - local auth_string= - - if [ "$DESTINATION_MAILS" == "" ]; then - return 0 - fi - - if [ "$_DEBUG" == "yes" ]; then - Logger "Debug mode, no warning mail will be sent." "NOTICE" - return 0 - fi - - # - if [ "$_QUICK_SYNC" == "2" ]; then - Logger "Current task is a quicksync task. Will not send any alert." "NOTICE" - return 0 - fi - # - - eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" - if [ $? != 0 ]; then - Logger "Cannot create [$ALERT_LOG_FILE]" "WARN" - mail_no_attachment=1 - else - mail_no_attachment=0 - fi - MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE") - if [ $ERROR_ALERT -eq 1 ]; then - subject="Error alert for $INSTANCE_ID" - elif [ $WARN_ALERT -eq 1 ]; then - subject="Warning alert for $INSTANCE_ID" - else - subject="Alert for $INSTANCE_ID" - fi - - if [ "$mail_no_attachment" -eq 0 ]; then - attachment_command="-a $ALERT_LOG_FILE" - fi - if type mutt > /dev/null 2>&1 ; then - echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command - if [ $? != 0 ]; then - Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN" - else - Logger "Sent alert mail using mutt." "NOTICE" - return 0 - fi - fi - - if type mail > /dev/null 2>&1 ; then - if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then - attachment_command="-A $ALERT_LOG_FILE" - elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then - attachment_command="-a$ALERT_LOG_FILE" - else - attachment_command="" - fi - echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS - if [ $? != 0 ]; then - Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN" - echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS - if [ $? != 0 ]; then - Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN" - else - Logger "Sent alert mail using mail command without attachment." "NOTICE" - return 0 - fi - else - Logger "Sent alert mail using mail command." "NOTICE" - return 0 - fi - fi - - if type sendmail > /dev/null 2>&1 ; then - echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS - if [ $? != 0 ]; then - Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN" - else - Logger "Sent alert mail using sendmail command without attachment." "NOTICE" - return 0 - fi - fi - - # Windows specific - if type "mailsend.exe" > /dev/null 2>&1 ; then - - if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then - Logger "Bogus smtp encryption, assuming none." "WARN" - encryption_string= - elif [ "$SMTP_ENCRYPTION" == "tls" ]; then - encryption_string=-starttls - elif [ "$SMTP_ENCRYPTION" == "ssl" ]:; then - encryption_string=-ssl - fi - if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then - auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\"" - fi - $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string - if [ $? != 0 ]; then - Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" - else - Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" - return 0 - fi - fi - - # Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/) - if type sendemail > /dev/null 2>&1 ; then - if [ "$SMTP_USER" != "" ] && [ "$SMTP_PASSWORD" != "" ]; then - SMTP_OPTIONS="-xu $SMTP_USER -xp $SMTP_PASSWORD" - else - SMTP_OPTIONS="" - fi - $(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 - if [ $? != 0 ]; then - Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN" - else - Logger "Sent alert mail using sendemail command without attachment." "NOTICE" - return 0 - fi - fi - - # pfSense specific - if [ -f /usr/local/bin/mail.php ]; then - echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject" - if [ $? != 0 ]; then - Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" - else - Logger "Sent alert mail using pfSense mail.php." "NOTICE" - return 0 - fi - fi - - # If function has not returned 0 yet, assume it is critical that no alert can be sent - Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue - - # Delete tmp log file - if [ -f "$ALERT_LOG_FILE" ]; then - rm "$ALERT_LOG_FILE" - fi -} - -# Generic email sending function. -# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or "" -# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" -# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend) -# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or "" -# smtp_server.domain.tld is mandatory, as is smtp_port (should be 25, 465 or 587) -# encryption can be set to tls, ssl or none -# smtp_user and smtp_password are optional -# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "sender_email@example.com" "smtp_server.domain.tld" "smtp_port" "encryption" "smtp_user" "smtp_password" -function SendEmail { - local subject="${1}" - local message="${2}" - local destination_mails="${3}" - local attachment="${4}" - local sender_email="${5}" - local smtp_server="${6}" - local smtp_port="${7}" - local encryption="${8}" - local smtp_user="${9}" - local smtp_password="${10}" - - # CheckArguments will report a warning that can be ignored if used in Windows with paranoia debug enabled - __CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local mail_no_attachment= - local attachment_command= - - local encryption_string= - local auth_string= - - if [ ! -f "$attachment" ]; then - attachment_command="-a $ALERT_LOG_FILE" - mail_no_attachment=1 - else - mail_no_attachment=0 - fi - - if type mutt > /dev/null 2>&1 ; then - echo "$message" | $(type -p mutt) -x -s "$subject" "$destination_mails" $attachment_command - if [ $? != 0 ]; then - Logger "Cannot send mail via $(type -p mutt) !!!" "WARN" - else - Logger "Sent mail using mutt." "NOTICE" - return 0 - fi - fi - - if type mail > /dev/null 2>&1 ; then - if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then - attachment_command="-A $attachment" - elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then - attachment_command="-a$attachment" - else - attachment_command="" - fi - echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destination_mails" - if [ $? != 0 ]; then - Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" - echo "$message" | $(type -p mail) -s "$subject" "$destination_mails" - if [ $? != 0 ]; then - Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN" - else - Logger "Sent mail using mail command without attachment." "NOTICE" - return 0 - fi - else - Logger "Sent mail using mail command." "NOTICE" - return 0 - fi - fi - - if type sendmail > /dev/null 2>&1 ; then - echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destination_mails" - if [ $? != 0 ]; then - Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN" - else - Logger "Sent mail using sendmail command without attachment." "NOTICE" - return 0 - fi - fi - - # Windows specific - if type "mailsend.exe" > /dev/null 2>&1 ; then - if [ "$sender_email" == "" ]; then - Logger "Missing sender email." "ERROR" - return 1 - fi - if [ "$smtp_server" == "" ]; then - Logger "Missing smtp port." "ERROR" - return 1 - fi - if [ "$smtp_port" == "" ]; then - Logger "Missing smtp port, assuming 25." "WARN" - smtp_port=25 - fi - if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then - Logger "Bogus smtp encryption, assuming none." "WARN" - encryption_string= - elif [ "$encryption" == "tls" ]; then - encryption_string=-starttls - elif [ "$encryption" == "ssl" ]:; then - encryption_string=-ssl - fi - if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then - auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\"" - fi - $(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string - if [ $? != 0 ]; then - Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" - else - Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" - return 0 - fi - fi - - # pfSense specific - if [ -f /usr/local/bin/mail.php ]; then - echo "$message" | /usr/local/bin/mail.php -s="$subject" - if [ $? != 0 ]; then - Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" - else - Logger "Sent mail using pfSense mail.php." "NOTICE" - return 0 - fi - fi - - # If function has not returned 0 yet, assume it is critical that no alert can be sent - Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue -} - -function TrapError { - local job="$0" - local line="$1" - local code="${2:-1}" - if [ $_SILENT -eq 0 ]; then - echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" - fi -} - -function LoadConfigFile { - local config_file="${1}" - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - - if [ ! -f "$config_file" ]; then - Logger "Cannot load configuration file [$config_file]. Cannot start." "CRITICAL" - exit 1 - elif [[ "$1" != *".conf" ]]; then - Logger "Wrong configuration file supplied [$config_file]. Cannot start." "CRITICAL" - exit 1 - else - grep '^[^ ]*=[^;&]*' "$config_file" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" # WITHOUT COMMENTS - # Shellcheck source=./sync.conf - source "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" - fi - - CONFIG_FILE="$config_file" -} - -function Spinner { - if [ $_SILENT -eq 1 ]; then - return 0 - fi - - case $toggle - in - 1) - echo -n " \ " - echo -ne "\r" - toggle="2" - ;; - - 2) - echo -n " | " - echo -ne "\r" - toggle="3" - ;; - - 3) - echo -n " / " - echo -ne "\r" - toggle="4" - ;; - - *) - echo -n " - " - echo -ne "\r" - toggle="1" - ;; - esac -} - -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} - -# Time control function for background processes, suitable for multiple synchronous processes -# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 -# Warning: Don't imbricate this function into another run if you plan to use the global variable output -function WaitForTaskCompletion { - local pids="${1}" # pids to wait for, separated by semi-colon - local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. - local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. - local caller_name="${4}" # Who called this function - local exit_on_error="${5:-false}" # Should the function exit on subprocess errors - local counting="${6:-true}" # Count time since function launch if true, script launch if false - - Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once - local log_ttime=0 # local time instance for comparaison - - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function - - local retval=0 # return value of monitored pid process - local errorcount=0 # Number of pids that finished with errors - - local pidCount # number of given pids - local pidState # State of the process - - IFS=';' read -a pidsArray <<< "$pids" - pidCount=${#pidsArray[@]} - - WAIT_FOR_TASK_COMPLETION="" - - #TODO: need to find a way to properly handle processes in unterruptible sleep state - - while [ ${#pidsArray[@]} -gt 0 ]; do - newPidsArray=() - for pid in "${pidsArray[@]}"; do - if kill -0 $pid > /dev/null 2>&1; then - # Handle uninterruptible sleep state or zombies by ommiting them from running process array - #TODO(high): have this tested on *BSD, Mac & Win - pidState=$(ps -p$pid -o state=) - if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then - newPidsArray+=($pid) - fi - else - wait $pid - result=$? - if [ $result -ne 0 ]; then - errorcount=$((errorcount+1)) - Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" - if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then - WAIT_FOR_TASK_COMPLETION="$pid:$result" - else - WAIT_FOR_TASK_COMPLETION=";$pid:$result" - fi - fi - fi - done - - Spinner - if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) - else - exec_time=$SECONDS - fi - - if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then - if [ $log_ttime -ne $exec_time ]; then - log_ttime=$exec_time - Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" - fi - fi - - if [ $exec_time -gt $soft_max_time ]; then - if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then - Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" - soft_alert=1 - SendAlert - - fi - if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then - Logger "Max hard execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" - KillChilds $pid - if [ $? == 0 ]; then - Logger "Task stopped successfully." "NOTICE" - else - Logger "Could not stop task." "ERROR" - fi - SendAlert - errrorcount=$((errorcount+1)) - fi - fi - - pidsArray=("${newPidsArray[@]}") - sleep $SLEEP_TIME - done - - Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then - Logger "Stopping execution." "CRITICAL" - exit 1337 - else - return $errorcount - fi -} - -function CleanUp { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - if [ "$_DEBUG" != "yes" ]; then - rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID" - # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) - rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.tmp" - fi -} - -#### MINIMAL-FUNCTION-SET END #### - -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - -# Usage: var=$(StripSingleQuotes "$var") -function StripSingleQuotes { - local string="${1}" - string="${string/#\'/}" # Remove singlequote if it begins string - string="${string/%\'/}" # Remove singlequote if it ends string - echo "$string" -} - -# Usage: var=$(StripDoubleQuotes "$var") -function StripDoubleQuotes { - local string="${1}" - string="${string/#\"/}" - string="${string/%\"/}" - echo "$string" -} - -function StripQuotes { - local string="${1}" - echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" -} - -function EscapeSpaces { - local string="${1}" # String on which spaces will be escaped - echo "${string// /\ }" -} - -function IsNumeric { - eval "local value=\"${1}\"" # Needed so variable variables can be processed - - local re="^-?[0-9]+([.][0-9]+)?$" - if [[ $value =~ $re ]]; then - echo 1 - else - echo 0 - fi -} - -## from https://gist.github.com/cdown/1163649 -function urlEncode { - local length="${#1}" - - local LANG=C - for (( i = 0; i < length; i++ )); do - local c="${1:i:1}" - case $c in - [a-zA-Z0-9.~_-]) - printf "$c" - ;; - *) - printf '%%%02X' "'$c" - ;; - esac - done -} - -function urlDecode { - local url_encoded="${1//+/ }" - - printf '%b' "${url_encoded//%/\\x}" -} - -function GetLocalOS { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local local_os_var= - - local_os_var="$(uname -spio 2>&1)" - if [ $? != 0 ]; then - local_os_var="$(uname -v 2>&1)" - if [ $? != 0 ]; then - local_os_var="$(uname)" - fi - fi - - case $local_os_var in - *"Linux"*) - LOCAL_OS="Linux" - ;; - *"BSD"*) - LOCAL_OS="BSD" - ;; - *"MINGW32"*|*"CYGWIN"*) - LOCAL_OS="msys" - ;; - *"Darwin"*) - LOCAL_OS="MacOSX" - ;; - *) - if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option - Logger "Running on unknown local OS [$local_os_var]." "WARN" - return - fi - Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" - exit 1 - ;; - esac - Logger "Local OS: [$local_os_var]." "DEBUG" -} - -function GetRemoteOS { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local cmd= - local remote_os_var= - - - if [ "$REMOTE_OPERATION" == "yes" ]; then - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" & - WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-1" false true - retval=$? - if [ $retval != 0 ]; then - cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" & - WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-2" false true - retval=$? - if [ $retval != 0 ]; then - cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" & - WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-3" false true - retval=$? - if [ $retval != 0 ]; then - Logger "Cannot Get remote OS type." "ERROR" - fi - fi - fi - - remote_os_var=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") - - case $remote_os_var in - *"Linux"*) - REMOTE_OS="Linux" - ;; - *"BSD"*) - REMOTE_OS="BSD" - ;; - *"MINGW32"*|*"CYGWIN"*) - REMOTE_OS="msys" - ;; - *"Darwin"*) - REMOTE_OS="MacOSX" - ;; - *"ssh"*|*"SSH"*) - Logger "Cannot connect to remote system." "CRITICAL" - exit 1 - ;; - *) - if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option - Logger "Running on unknown remote OS [$remote_os_var]." "WARN" - return - fi - Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" - Logger "Remote OS said:\n$remote_os_var" "CRITICAL" - exit 1 - esac - - Logger "Remote OS: [$remote_os_var]." "DEBUG" - fi -} - -function RunLocalCommand { - local command="${1}" # Command to run - local hard_max_time="${2}" # Max time to wait for command to compleet - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - if [ $_DRYRUN -ne 0 ]; then - Logger "Dryrun: Local command [$command] not run." "NOTICE" - return 0 - fi - - Logger "Running command [$command] on local host." "NOTICE" - eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & - WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} false true - retval=$? - if [ $retval -eq 0 ]; then - Logger "Command succeded." "NOTICE" - else - Logger "Command failed." "ERROR" - fi - - if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" - fi - - if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then - Logger "Stopping on command execution error." "CRITICAL" - exit 1 - fi -} - -## Runs remote command $1 and waits for completition in $2 seconds -function RunRemoteCommand { - local command="${1}" # Command to run - local hard_max_time="${2}" # Max time to wait for command to compleet - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - if [ $_DRYRUN -ne 0 ]; then - Logger "Dryrun: Local command [$command] not run." "NOTICE" - return 0 - fi - - Logger "Running command [$command] on remote host." "NOTICE" - cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" & - WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} false true - retval=$? - if [ $retval -eq 0 ]; then - Logger "Command succeded." "NOTICE" - else - Logger "Command failed." "ERROR" - fi - - if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]) - then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" - fi - - if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then - Logger "Stopping on command execution error." "CRITICAL" - exit 1 - fi -} - -function RunBeforeHook { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local pids= - - if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then - RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & - pids="$!" - fi - - if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then - RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & - pids="$pids;$!" - fi - if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false true - fi -} - -function RunAfterHook { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local pids - - if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then - RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & - pids="$!" - fi - - if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then - RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & - pids="$pids;$!" - fi - if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false true - fi -} - -function CheckConnectivityRemoteHost { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug - - if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then - eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & - WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} false true - if [ $? != 0 ]; then - Logger "Cannot ping $REMOTE_HOST" "ERROR" - return 1 - fi - fi - fi -} - -function CheckConnectivity3rdPartyHosts { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local remote_3rd_party_success - local pids - - if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug - - if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then - remote_3rd_party_success=0 - for i in $REMOTE_3RD_PARTY_HOSTS - do - eval "$PING_CMD $i > /dev/null 2>&1" & - WaitForTaskCompletion $! 10 360 ${FUNCNAME[0]} false true - if [ $? != 0 ]; then - Logger "Cannot ping 3rd party host $i" "NOTICE" - else - remote_3rd_party_success=1 - fi - done - - if [ $remote_3rd_party_success -ne 1 ]; then - Logger "No remote 3rd party host responded to ping. No internet ?" "ERROR" - return 1 - else - return 0 - fi - fi - fi -} - -#__BEGIN_WITH_PARANOIA_DEBUG -function __CheckArguments { - # Checks the number of arguments of a function and raises an error if some are missing - - if [ "$_DEBUG" == "yes" ]; then - local number_of_arguments="${1}" # Number of arguments the tested function should have - local number_of_given_arguments="${2}" # Number of arguments that have been passed - local function_name="${3}" # Function name that called __CheckArguments - - if [ "$_PARANOIA_DEBUG" == "yes" ]; then - Logger "Entering function [$function_name]." "DEBUG" - fi - - # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) - # If any of the arguments contains spaces, bash things there are two aguments - # In order to avoid this, we need to iterate over ${4} and count - - local iterate=4 - local fetch_arguments=1 - local arg_list="" - while [ $fetch_arguments -eq 1 ]; do - cmd='argument=${'$iterate'}' - eval $cmd - if [ "$argument" = "" ]; then - fetch_arguments=0 - else - arg_list="$arg_list [Argument $(($iterate-3)): $argument]" - iterate=$(($iterate+1)) - fi - done - local counted_arguments=$((iterate-4)) - - if [ $counted_arguments -ne $number_of_arguments ]; then - Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR" - Logger "Arguments passed: $arg_list" "ERROR" - fi - fi -} - -#__END_WITH_PARANOIA_DEBUG - -function RsyncPatternsAdd { - local pattern_type="${1}" # exclude or include - local pattern="${2}" - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local rest - - # Disable globbing so wildcards from exclusions do not get expanded - set -f - rest="$pattern" - while [ -n "$rest" ] - do - # Take the string until first occurence until $PATH_SEPARATOR_CHAR - str=${rest%%;*} - # Handle the last case - if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then - rest= - else - # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR - rest=${rest#*$PATH_SEPARATOR_CHAR} - fi - if [ "$RSYNC_PATTERNS" == "" ]; then - RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" - else - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" - fi - done - set +f -} - -function RsyncPatternsFromAdd { - local pattern_type="${1}" - local pattern_from="${2}" - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - ## Check if the exclude list has a full path, and if not, add the config file path if there is one - if [ "$(basename $pattern_from)" == "$pattern_from" ]; then - pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" - fi - - if [ -e "$pattern_from" ]; then - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" - fi -} - -function RsyncPatterns { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then - if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then - RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" - fi - if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then - RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" - fi - if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then - RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" - fi - if [ "$RSYNC_INCLUDE_FROM" != "" ]; then - RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" - fi - elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then - if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then - RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" - fi - if [ "$RSYNC_INCLUDE_FROM" != "" ]; then - RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" - fi - if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then - RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" - fi - if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then - RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" - fi - else - Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" - fi -} - -function PreInit { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - ## SSH compression - if [ "$SSH_COMPRESSION" != "no" ]; then - SSH_COMP=-C - else - SSH_COMP= - fi - - ## Ignore SSH known host verification - if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then - SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - fi - - ## Support for older config files without RSYNC_EXECUTABLE option - if [ "$RSYNC_EXECUTABLE" == "" ]; then - RSYNC_EXECUTABLE=rsync - fi - - ## Sudo execution option - if [ "$SUDO_EXEC" == "yes" ]; then - if [ "$RSYNC_REMOTE_PATH" != "" ]; then - RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" - else - RSYNC_PATH="sudo $RSYNC_EXECUTABLE" - fi - COMMAND_SUDO="sudo" - else - if [ "$RSYNC_REMOTE_PATH" != "" ]; then - RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" - else - RSYNC_PATH="$RSYNC_EXECUTABLE" - fi - COMMAND_SUDO="" - fi - - ## Set rsync default arguments - RSYNC_ARGS="-rltD" - if [ "$_DRYRUN" -eq 1 ]; then - RSYNC_DRY_ARG="-n" - DRY_WARNING="/!\ DRY RUN" - else - RSYNC_DRY_ARG="" - fi - - RSYNC_ATTR_ARGS="" - if [ "$PRESERVE_PERMISSIONS" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" - fi - if [ "$PRESERVE_OWNER" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" - fi - if [ "$PRESERVE_GROUP" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" - fi - if [ "$PRESERVE_ACL" == "yes" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" - fi - if [ "$PRESERVE_XATTR" == "yes" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" - fi - if [ "$RSYNC_COMPRESS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -z" - fi - if [ "$COPY_SYMLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -L" - fi - if [ "$KEEP_DIRLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -K" - fi - if [ "$PRESERVE_HARDLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -H" - fi - if [ "$CHECKSUM" == "yes" ]; then - RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" - fi - if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" - fi - - if [ "$PARTIAL" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" - fi - - if [ "$DELTA_COPIES" != "no" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" - else - RSYNC_ARGS=$RSYNC_ARGS" --whole-file" - fi - - ## Set compression executable and extension - COMPRESSION_LEVEL=3 - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" -} - -function PostInit { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - # Define remote commands - SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" -} - -function InitLocalOSSettings { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - ## If running under Msys, some commands do not run the same way - ## Using mingw version of find instead of windows one - ## Getting running processes is quite different - ## Ping command is not the same - if [ "$LOCAL_OS" == "msys" ]; then - FIND_CMD=$(dirname $BASH)/find - PING_CMD='$SYSTEMROOT\system32\ping -n 2' - else - FIND_CMD=find - PING_CMD="ping -c 2 -i .2" - fi - - ## Stat command has different syntax on Linux and FreeBSD/MacOSX - if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then - STAT_CMD="stat -f \"%Sm\"" - STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" - else - STAT_CMD="stat --format %y" - STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" - fi -} - -function InitRemoteOSSettings { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - #TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes - ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - if [ "$PRESERVE_EXECUTABILITY" != "no" ];then - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" - fi - fi - - if [ "$REMOTE_OS" == "msys" ]; then - REMOTE_FIND_CMD=$(dirname $BASH)/find - else - REMOTE_FIND_CMD=find - fi - - ## Stat command has different syntax on Linux and FreeBSD/MacOSX - if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then - REMOTE_STAT_CMD="stat -f \"%Sm\"" - REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" - else - REMOTE_STAT_CMD="stat --format %y" - REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" - fi - -} - -## END Generic functions diff --git a/osync.sh b/osync.sh index d3a1412..fd69046 100755 --- a/osync.sh +++ b/osync.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash #TODO(critical): handle conflict prevalance, especially in sync_attrs function -#TODO(critical): test new WaitForTaskCompletion behavior with self=true +#TODO(critical): writelockfiles remote does not shut execution PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016081902 +PROGRAM_BUILD=2016082201 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016081805 +## FUNC_BUILD=2016082204 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -184,7 +184,7 @@ function KillChilds { # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then Logger "Sending SIGTERM to process [$pid]." "DEBUG" - kill -s SIGTERM "$pid" + kill -s TERM "$pid" if [ $? != 0 ]; then sleep 15 Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" @@ -594,29 +594,6 @@ function WaitForTaskCompletion { while [ ${#pidsArray[@]} -gt 0 ]; do newPidsArray=() - for pid in "${pidsArray[@]}"; do - if kill -0 $pid > /dev/null 2>&1; then - # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) - #TODO(high): have this tested on *BSD, Mac & Win - pidState=$(ps -p$pid -o state= 2 > /dev/null) - if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then - newPidsArray+=($pid) - fi - else - # pid is dead, get it's exit code from wait command - wait $pid - retval=$? - if [ $retval -ne 0 ]; then - errorcount=$((errorcount+1)) - Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" - if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then - WAIT_FOR_TASK_COMPLETION="$pid:$result" - else - WAIT_FOR_TASK_COMPLETION=";$pid:$result" - fi - fi - fi - done Spinner if [ $counting == true ]; then @@ -656,6 +633,30 @@ function WaitForTaskCompletion { fi fi + for pid in "${pidsArray[@]}"; do + if kill -0 $pid > /dev/null 2>&1; then + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + #TODO(high): have this tested on *BSD, Mac & Win + pidState=$(ps -p$pid -o state= 2 > /dev/null) + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + newPidsArray+=($pid) + fi + else + # pid is dead, get it's exit code from wait command + wait $pid + retval=$? + if [ $retval -ne 0 ]; then + errorcount=$((errorcount+1)) + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then + WAIT_FOR_TASK_COMPLETION="$pid:$result" + else + WAIT_FOR_TASK_COMPLETION=";$pid:$result" + fi + fi + fi + done + pidsArray=("${newPidsArray[@]}") sleep $SLEEP_TIME done @@ -975,7 +976,7 @@ function CheckConnectivity3rdPartyHosts { for i in $REMOTE_3RD_PARTY_HOSTS do eval "$PING_CMD $i > /dev/null 2>&1" & - WaitForTaskCompletion $! 10 360 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot ping 3rd party host $i" "NOTICE" else @@ -1310,9 +1311,7 @@ function TrapQuit { exitcode=2 # Warning exit code must not force daemon mode to quit else UnlockReplicas - if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then - RunAfterHook - fi + RunAfterHook CleanUp Logger "$PROGRAM finished." "NOTICE" exitcode=0 @@ -1429,15 +1428,16 @@ function CheckReplicaPaths { local pids + # Use direct comparaison before having a portable realpath implementation #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO(verylow): investigate realpath & readlink issues on MSYS and busybox here #TARGET_SYNC_DIR_CANN=$(realpath "${TARGET[1]}) - #if [ "$REMOTE_OPERATION" != "yes" ]; then - # if [ "$INITIATOR_SYNC_DIR_CANN" == "$TARGET_SYNC_DIR_CANN" ]; then - # Logger "Master directory [${INITIATOR[1]}] cannot be the same as target directory." "CRITICAL" - # exit 1 - # fi - #fi + if [ "$REMOTE_OPERATION" != "yes" ]; then + if [ "${INITIATOR[1]}" == "${TARGET[1]}" ]; then + Logger "Initiator and target path [${INITIATOR[1]}] cannot be the same." "CRITICAL" + exit 1 + fi + fi _CheckReplicaPathsLocal "${INITIATOR[1]}" & pids="$!" @@ -2746,10 +2746,10 @@ function Init { # Do not use exit and quit traps if osync runs in monitor mode if [ $sync_on_changes -eq 0 ]; then - trap TrapStop SIGINT SIGHUP SIGTERM SIGQUIT + trap TrapStop INT HUP TERM QUIT trap TrapQuit EXIT else - trap TrapQuit SIGTERM EXIT SIGHUP SIGQUIT + trap TrapQuit TERM EXIT HUP QUIT fi local uri From 6f921205f5dced03e6251de123a67e7b10e61778 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 10:26:38 +0200 Subject: [PATCH 09/16] Added basic unit tests --- dev/tests/run.sh | 354 ++++++ dev/tests/run_tests.sh | 54 + dev/tests/shunit2/shunit2 | 1067 ++++++++++++++++++ dev/tests/shunit2/shunit2_test.sh | 124 ++ dev/tests/shunit2/shunit2_test_asserts.sh | 206 ++++ dev/tests/shunit2/shunit2_test_failures.sh | 86 ++ dev/tests/shunit2/shunit2_test_helpers | 229 ++++ dev/tests/shunit2/shunit2_test_macros.sh | 246 ++++ dev/tests/shunit2/shunit2_test_misc.sh | 160 +++ dev/tests/shunit2/shunit2_test_standalone.sh | 41 + 10 files changed, 2567 insertions(+) create mode 100755 dev/tests/run.sh create mode 100755 dev/tests/run_tests.sh create mode 100755 dev/tests/shunit2/shunit2 create mode 100755 dev/tests/shunit2/shunit2_test.sh create mode 100755 dev/tests/shunit2/shunit2_test_asserts.sh create mode 100755 dev/tests/shunit2/shunit2_test_failures.sh create mode 100644 dev/tests/shunit2/shunit2_test_helpers create mode 100755 dev/tests/shunit2/shunit2_test_macros.sh create mode 100755 dev/tests/shunit2/shunit2_test_misc.sh create mode 100755 dev/tests/shunit2/shunit2_test_standalone.sh diff --git a/dev/tests/run.sh b/dev/tests/run.sh new file mode 100755 index 0000000..4448ec1 --- /dev/null +++ b/dev/tests/run.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash + +# osync shunit2 tests + +# modify ctime on ext4 +debugfs -w -R 'set_inode_field /tmp/foo ctime 201001010101' /dev/sda1 +echo > /proc/sys/vm/drop_caches + +# Test dir +TMP="/tmp/osync_tests" +# SSH port used for remote tests +SSH_PORT=22 + +INITIATOR_DIR="init" +TARGET_DIR="targ" + +# Get dir the tests are stored in +TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +cd "$TEST_DIR" + +OSYNC_EXECUTABLE="$(dirname $TEST_DIR)//osync.sh" +declare -A sandbox_osync +sandbox_osync[quickLocal]="--initiator=$INITIATOR_DIR --target=$TARGET_DIR" +#sandbox_osync[quickRemote]="--initiator=$INITIATOR_DIR --slave=ssh://localhost:$SSH_PORT/$TMP/$TARGET_DIR" +#sandbox_osync[local]="conf/local.conf" +#sandbox_osync[remote]="conf/remote.conf" + +oneTimeSetUp() +{ + for i in "${!sandbox_osync[@]}" + do + prepareSandbox "$i" + done +} + +oneTimeTearDown() +{ + rm -rf "$TMP" +} + +prepareSandbox() +{ + rm -rf "$TMP/$1" + mkdir -p "$TMP/$1" + pushd "$TMP/$1" >/dev/null + mkdir "$INITIATOR_DIR" + mkdir "$TARGET_DIR" + mkdir expected + popd >/dev/null +} + +compareSandbox() +{ + diff -aurx .osync_workdir master slave + assertEquals 0 $? + + diff -aurx .osync_workdir master expected + assertEquals 0 $? + + diff -aurx .osync_workdir slave expected + assertEquals 0 $? +} + +syncSandbox() +{ + $OSYNC_EXECUTABLE ${sandbox_osync[$1]} >/dev/null + assertEquals 0 $? +} + +runSandbox() +{ + syncSandbox "$1" + compareSandbox +} + +joinSandbox() +{ + cd "$TMP/$1" +} + +### Tests ### +# One empty file +_testOneEmptyFile() +{ + joinSandbox "$1" + + # Add one empty file + touch "$2/testOneEmpty" + touch expected/testOneEmpty + runSandbox "$1" + + # Change one empty file + echo "Test" > "$2/testOneEmpty" + cp "$2/testOneEmpty" expected/testOneEmpty + runSandbox "$1" + + # Empty one file + echo -n "" > "$2/testOneEmpty" + cp "$2/testOneEmpty" expected/testOneEmpty + runSandbox "$1" + + # Delete one empty file + cp "$2/testOneEmpty" testOneEmpty + rm "$2/testOneEmpty" + rm expected/testOneEmpty + runSandbox "$1" + # Backup check + if [ "$2" == "master" ] + then + diff -aur slave/.osync_workdir/deleted/testOneEmpty testOneEmpty + else + diff -aur master/.osync_workdir/deleted/testOneEmpty testOneEmpty + fi + assertEquals 0 $? +} + +testQuickLocalMasterOneEmptyFile() +{ + _testOneEmptyFile quickLocal master +} + +testQuickLocalSlaveOneEmptyFile() +{ + _testOneEmptyFile quickLocal slave +} + +testQuickRemoteMasterOneEmptyFile() +{ + _testOneEmptyFile quickRemote master +} + +testQuickRemoteSlaveOneEmptyFile() +{ + _testOneEmptyFile quickRemote slave +} + +testLocalMasterOneEmptyFile() +{ + _testOneEmptyFile local master +} + +testLocalSlaveOneEmptyFile() +{ + _testOneEmptyFile local slave +} + +testRemoteMasterOneEmptyFile() +{ + _testOneEmptyFile remote master +} + +testRemoteSlaveOneEmptyFile() +{ + _testOneEmptyFile remote slave +} + +# One file +_testOneFile() +{ + joinSandbox "$1" + + # Add one file + echo "Test" > "$2/testOne" + cp "$2/testOne" expected/testOne + runSandbox "$1" + + # Change one file + echo "Test2" > "$2/testOne" + cp "$2/testOne" expected/testOne + runSandbox "$1" + + # Delete one file + cp "$2/testOne" testOne + rm "$2/testOne" + rm expected/testOne + runSandbox "$1" + # Backup check + if [ "$2" == "master" ] + then + diff -aur slave/.osync_workdir/deleted/testOne testOne + else + diff -aur master/.osync_workdir/deleted/testOne testOne + fi + assertEquals 0 $? +} + +testQuickLocalMasterOneFile() +{ + _testOneFile quickLocal master +} + +testQuickLocalSlaveOneFile() +{ + _testOneFile quickLocal slave +} + +testQuickRemoteMasterOneFile() +{ + _testOneFile quickRemote master +} + +testQuickRemoteSlaveOneFile() +{ + _testOneFile quickRemote slave +} + +testLocalMasterOneFile() +{ + _testOneFile local master +} + +testLocalSlaveOneFile() +{ + _testOneFile local slave +} + +testRemoteMasterOneFile() +{ + _testOneFile remote master +} + +testRemoteSlaveOneFile() +{ + _testOneFile remote slave +} + +# Distinct +_testDistinct() +{ + joinSandbox "$1" + + # Generate files in master + for i in testDistinctM1 testDistinctM2 testDistinctM3 + do + mkdir "master/$i" + mkdir "expected/$i" + for j in m1 m2 m3 ; do + echo "$i/$j" > "master/$i/$j" + cp "master/$i/$j" "expected/$i/$j" + done + done + + # Generate files in slave + for i in testDistinctS1 testDistinctS2 testDistinctS3 + do + mkdir "slave/$i" + mkdir "expected/$i" + for j in s1 s2 s3 ; do + echo "$i/$j" > "slave/$i/$j" + cp "slave/$i/$j" "expected/$i/$j" + done + done + + # Generate files in same directories for master and slave + for i in testDistinctMS1 testDistinctMS2 testDistinctMS3 + do + mkdir "master/$i" + mkdir "slave/$i" + mkdir "expected/$i" + for j in ms1 ms2 ms3 ; do + echo "$i/$j" > "master/$i/m-$j" + cp "master/$i/m-$j" "expected/$i/m-$j" + echo "$i/$j" > "slave/$i/s-$j" + cp "slave/$i/s-$j" "expected/$i/s-$j" + done + done + + runSandbox "$1" +} + +testQuickLocalDistinct() +{ + _testDistinct quickLocal +} + +testQuickRemoteDistinct() +{ + _testDistinct quickRemote +} + +testLocalDistinct() +{ + _testDistinct local +} + +testRemoteDistinct() +{ + _testDistinct remote +} + +# Collision +_testCollision() +{ + joinSandbox "$1" + + # Slave precedence + echo "Test1" > master/testCollision1 + echo "Test2" > slave/testCollision1 + touch -d "2004-02-29 16:21:41" master/testCollision1 + touch -d "2004-02-29 16:21:42" slave/testCollision1 + cp slave/testCollision1 expected/testCollision1 + cp master/testCollision1 testCollision1 + runSandbox "$1" + # Backup check + diff -aur master/.osync_workdir/backups/testCollision1 testCollision1 + assertEquals 0 $? + + # Master precedence + echo "Test1" > master/testCollision2 + echo "Test2" > slave/testCollision2 + touch -d "2004-02-29 16:21:42" master/testCollision2 + touch -d "2004-02-29 16:21:41" slave/testCollision2 + cp master/testCollision2 expected/testCollision2 + cp slave/testCollision2 testCollision2 + runSandbox "$1" + # Backup check + diff -aur slave/.osync_workdir/backups/testCollision2 testCollision2 + assertEquals 0 $? + + # ?? +# echo "Test1" > master/testCollision3 +# echo "Test2" > slave/testCollision3 +# touch -d "2004-02-29 16:21:42" master/testCollision3 +# touch -d "2004-02-29 16:21:42" slave/testCollision3 +# cp slave/testCollision3 expected/testCollision3 +# runSandbox "$1" +} + +testQuickLocalCollision() +{ + _testCollision quickLocal +} + +testQuickRemoteCollision() +{ + _testCollision quickRemote +} + +testLocalCollision() +{ + _testCollision local +} + +testRemoteCollision() +{ + _testCollision remote +} + +#suite() +#{ +# suite_addTest "testQuickRemoteMasterOneEmptyFile" +#} + +. shunit2/shunit2 diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh new file mode 100755 index 0000000..3f88d6a --- /dev/null +++ b/dev/tests/run_tests.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# osync test suite 2016081901 + +DEV_DIR="/home/git/osync/dev" +OSYNC_EXECUTABLE="n_osync.sh" + +INITIATOR_DIR="/opt/osync/initiator" +TARGET_DIR="/opt/osync/target" +OSYNC_STATE_DIR=".osync_workdir/state" + +function CreateReplicas () { + if [ -d "$INITIATOR_DIR" ]; then + rm -rf "$INITIATOR_DIR" + fi + mkdir -p "$INITIATOR_DIR" + + if [ -d "$TARGET_DIR" ]; then + rm -rf "$TARGET_DIR" + fi + mkdir -p "$TARGET_DIR" +} + +function oneTimeSetUp () { + source "$DEV_DIR/ofunctions.sh" + + if grep "^IS_STABLE=YES" "$DEV_DIR/$OSYNC_EXECUTABLE" > /dev/null; then + IS_STABLE=yes + else + IS_STABLE=no + sed -i 's/^IS_STABLE=no/IS_STABLE=yes/' "$DEV_DIR/$OSYNC_EXECUTABLE" + fi +} + +function oneTimeTearDown () { + if [ "$IS_STABLE" == "no" ]; then + sed -i 's/^IS_STABLE=yes/IS_STABLE=no/' "$DEV_DIR/$OSYNC_EXECUTABLE" + fi +} + +function test_osync_quicksync_local () { + CreateReplicas + cd "$DEV_DIR" + ./n_osync.sh --initiator="$INITIATOR_DIR" --target="$TARGET_DIR" > /dev/null + assertEquals "Return code" "0" $? + + [ -d "$INITIATOR_DIR/$OSYNC_STATE_DIR" ] + assertEquals "Initiator state dir exists" "0" $? + + [ -d "$TARGET_DIR/$OSYNC_STATE_DIR" ] + assertEquals "Target state dir exists" "0" $? +} + +. ./shunit2/shunit2 diff --git a/dev/tests/shunit2/shunit2 b/dev/tests/shunit2/shunit2 new file mode 100755 index 0000000..d6e7503 --- /dev/null +++ b/dev/tests/shunit2/shunit2 @@ -0,0 +1,1067 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# +# shUnit2 -- Unit testing framework for Unix shell scripts. +# http://code.google.com/p/shunit2/ +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is +# based on the popular JUnit unit testing framework for Java. + +# return if shunit already loaded +[ -n "${SHUNIT_VERSION:-}" ] && exit 0 +SHUNIT_VERSION='2.1.7pre' + +# return values that scripts can use +SHUNIT_TRUE=0 +SHUNIT_FALSE=1 +SHUNIT_ERROR=2 + +# logging functions +_shunit_warn() { echo "shunit2:WARN $@" >&2; } +_shunit_error() { echo "shunit2:ERROR $@" >&2; } +_shunit_fatal() { echo "shunit2:FATAL $@" >&2; exit ${SHUNIT_ERROR}; } + +# determine some reasonable command defaults +__SHUNIT_UNAME_S=`uname -s` +case "${__SHUNIT_UNAME_S}" in + BSD) __SHUNIT_EXPR_CMD='gexpr' ;; + *) __SHUNIT_EXPR_CMD='expr' ;; +esac + +# commands a user can override if needed +SHUNIT_EXPR_CMD=${SHUNIT_EXPR_CMD:-${__SHUNIT_EXPR_CMD}} + +# enable strict mode by default +SHUNIT_STRICT=${SHUNIT_STRICT:-${SHUNIT_TRUE}} + +# specific shell checks +if [ -n "${ZSH_VERSION:-}" ]; then + setopt |grep "^shwordsplit$" >/dev/null + if [ $? -ne ${SHUNIT_TRUE} ]; then + _shunit_fatal 'zsh shwordsplit option is required for proper operation' + fi + if [ -z "${SHUNIT_PARENT:-}" ]; then + _shunit_fatal "zsh does not pass \$0 through properly. please declare \ +\"SHUNIT_PARENT=\$0\" before calling shUnit2" + fi +fi + +# +# constants +# + +__SHUNIT_ASSERT_MSG_PREFIX='ASSERT:' +__SHUNIT_MODE_SOURCED='sourced' +__SHUNIT_MODE_STANDALONE='standalone' +__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} + +# set the constants readonly +__shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1` +echo "${__shunit_constants}" |grep '^Binary file' >/dev/null && \ + __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1` +for __shunit_const in ${__shunit_constants}; do + if [ -z "${ZSH_VERSION:-}" ]; then + readonly ${__shunit_const} + else + case ${ZSH_VERSION} in + [123].*) readonly ${__shunit_const} ;; + *) readonly -g ${__shunit_const} # declare readonly constants globally + esac + fi +done +unset __shunit_const __shunit_constants + +# +# internal variables +# + +# variables +__shunit_lineno='' # line number of executed test +__shunit_mode=${__SHUNIT_MODE_SOURCED} # operating mode +__shunit_reportGenerated=${SHUNIT_FALSE} # is report generated +__shunit_script='' # filename of unittest script (standalone mode) +__shunit_skip=${SHUNIT_FALSE} # is skipping enabled +__shunit_suite='' # suite of tests to execute + +# counts of tests +__shunit_testSuccess=${SHUNIT_TRUE} +__shunit_testsTotal=0 +__shunit_testsPassed=0 +__shunit_testsFailed=0 + +# counts of asserts +__shunit_assertsTotal=0 +__shunit_assertsPassed=0 +__shunit_assertsFailed=0 +__shunit_assertsSkipped=0 + +# macros +_SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' + +#----------------------------------------------------------------------------- +# private functions + +#----------------------------------------------------------------------------- +# assert functions +# + +# Assert that two values are equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertEquals() requires two or three arguments; $# given" + _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}${4:+ 4: $4}" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then + _shunit_assertPass + else + failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' + +# Assert that two values are not equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotEquals() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if [ "${shunit_expected_}" != "${shunit_actual_}" ]; then + _shunit_assertPass + else + failSame "${shunit_message_}" "$@" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' + +# Assert that a value is null (i.e. an empty string) +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNull() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertNull() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertTrue "${shunit_message_}" "[ -z '$1' ]" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' + +# Assert that a value is not null (i.e. a non-empty string) +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotNull() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null + _shunit_error "assertNotNull() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` + test -n "${shunit_actual_}" + assertTrue "${shunit_message_}" $? + shunit_return=$? + + unset shunit_actual_ shunit_message_ + return ${shunit_return} +} +_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' + +# Assert that two values are the same (i.e. equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' + +# Assert that two values are not the same (i.e. not equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_:-}$1" + shift + fi + assertNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is true. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertTrue 0 +# assertTrue "[ 34 -gt 23 ]" +# The folloing test will fail with a message: +# assertTrue 123 +# assertTrue "test failed" "[ -r '/non/existant/file' ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertTrue() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertTrue() takes one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # see if condition is an integer, i.e. a return value + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if [ -z "${shunit_condition_}" ]; then + # null condition + shunit_return=${SHUNIT_FALSE} + elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] + then + # possible return value. treating 0 as true, and non-zero as false. + [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE} + else + # (hopefully) a condition + ( eval ${shunit_condition_} ) >/dev/null 2>&1 + [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # record the test + if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is false. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertFalse 1 +# assertFalse "[ 'apples' = 'oranges' ]" +# The folloing test will fail with a message: +# assertFalse 0 +# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertFalse() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertFalse() quires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # see if condition is an integer, i.e. a return value + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if [ -z "${shunit_condition_}" ]; then + # null condition + shunit_return=${SHUNIT_FALSE} + elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] + then + # possible return value. treating 0 as true, and non-zero as false. + [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE} + else + # (hopefully) a condition + ( eval ${shunit_condition_} ) >/dev/null 2>&1 + [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # record the test + if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# failure functions +# + +# Records a test failure. +# +# Args: +# message: string: failure message [optional] +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +fail() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 1 ]; then + _shunit_error "fail() requires zero or one arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 1 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_}" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +_FAIL_='eval fail --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${SHUNIT_FALSE} +} +_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' + +# Records a test failure, stating two values should have been the same. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# This is functionally equivalent to calling failNotEquals(). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + failNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# skipping functions +# + +# Force remaining assert and fail functions to be "skipped". +# +# This function forces the remaining assert and fail functions to be "skipped", +# i.e. they will have no effect. Each function skipped will be recorded so that +# the total of asserts and fails will not be altered. +# +# Args: +# None +startSkipping() +{ + __shunit_skip=${SHUNIT_TRUE} +} + +# Resume the normal recording behavior of assert and fail calls. +# +# Args: +# None +endSkipping() +{ + __shunit_skip=${SHUNIT_FALSE} +} + +# Returns the state of assert and fail call skipping. +# +# Args: +# None +# Returns: +# boolean: (TRUE/FALSE constant) +isSkipping() +{ + return ${__shunit_skip} +} + +#----------------------------------------------------------------------------- +# suite functions +# + +# Stub. This function should contains all unit test calls to be made. +# +# DEPRECATED (as of 2.1.0) +# +# This function can be optionally overridden by the user in their test suite. +# +# If this function exists, it will be called when shunit2 is sourced. If it +# does not exist, shunit2 will search the parent script for all functions +# beginning with the word 'test', and they will be added dynamically to the +# test suite. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Adds a function name to the list of tests schedule for execution. +# +# This function should only be called from within the suite() function. +# +# Args: +# function: string: name of a function to add to current unit test suite +suite_addTest() +{ + shunit_func_=${1:-} + + __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" + __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` + + unset shunit_func_ +} + +# Stub. This function will be called once before any tests are run. +# +# Common one-time environment preparation tasks shared by all tests can be +# defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called once after all tests are finished. +# +# Common one-time environment cleanup tasks shared by all tests can be defined +# here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called before each test is run. +# +# Common environment preparation tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#setUp() { :; } + +# Note: see _shunit_mktempFunc() for actual implementation +# Stub. This function will be called after each test is run. +# +# Common environment cleanup tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +#------------------------------------------------------------------------------ +# internal shUnit2 functions +# + +# Create a temporary directory to store various run-time files in. +# +# This function is a cross-platform temporary directory creation tool. Not all +# OSes have the mktemp function, so one is included here. +# +# Args: +# None +# Outputs: +# string: the temporary directory that was created +_shunit_mktempDir() +{ + # try the standard mktemp function + ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return + + # the standard mktemp didn't work. doing our own. + if [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then + _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" +#! /bin/sh +exit ${SHUNIT_TRUE} +EOF + chmod +x "${_shunit_file_}" + done + + unset _shunit_file_ +} + +# Final cleanup function to leave things as we found them. +# +# Besides removing the temporary directory, this function is in charge of the +# final exit code of the unit test. The exit code is based on how the script +# was ended (e.g. normal exit, or via Ctrl-C). +# +# Args: +# name: string: name of the trap called (specified when trap defined) +_shunit_cleanup() +{ + _shunit_name_=$1 + + case ${_shunit_name_} in + EXIT) _shunit_signal_=0 ;; + INT) _shunit_signal_=2 ;; + TERM) _shunit_signal_=15 ;; + *) + _shunit_warn "unrecognized trap value (${_shunit_name_})" + _shunit_signal_=0 + ;; + esac + + # do our work + rm -fr "${__shunit_tmpDir}" + + # exit for all non-EXIT signals + if [ ${_shunit_name_} != 'EXIT' ]; then + _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" + # disable EXIT trap + trap 0 + # add 128 to signal and exit + exit `expr ${_shunit_signal_} + 128` + elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then + _shunit_assertFail 'Unknown failure encountered running a test' + _shunit_generateReport + exit ${SHUNIT_ERROR} + fi + + unset _shunit_name_ _shunit_signal_ +} + +# The actual running of the tests happens here. +# +# Args: +# None +_shunit_execSuite() +{ + for _shunit_test_ in ${__shunit_suite}; do + __shunit_testSuccess=${SHUNIT_TRUE} + + # disable skipping + endSkipping + + # execute the per-test setup function + setUp + + # execute the test + echo "${_shunit_test_}" + eval ${_shunit_test_} + + # execute the per-test tear-down function + tearDown + + # update stats + if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then + __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` + else + __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` + fi + done + + unset _shunit_test_ +} + +# Generates the user friendly report with appropriate OK/FAILED message. +# +# Args: +# None +# Output: +# string: the report of successful and failed tests, as well as totals. +_shunit_generateReport() +{ + _shunit_ok_=${SHUNIT_TRUE} + + # if no exit code was provided one, determine an appropriate one + [ ${__shunit_testsFailed} -gt 0 \ + -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ + && _shunit_ok_=${SHUNIT_FALSE} + + echo + if [ ${__shunit_testsTotal} -eq 1 ]; then + echo "Ran ${__shunit_testsTotal} test." + else + echo "Ran ${__shunit_testsTotal} tests." + fi + + _shunit_failures_='' + _shunit_skipped_='' + [ ${__shunit_assertsFailed} -gt 0 ] \ + && _shunit_failures_="failures=${__shunit_assertsFailed}" + [ ${__shunit_assertsSkipped} -gt 0 ] \ + && _shunit_skipped_="skipped=${__shunit_assertsSkipped}" + + if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then + _shunit_msg_='OK' + [ -n "${_shunit_skipped_}" ] \ + && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})" + else + _shunit_msg_="FAILED (${_shunit_failures_}" + [ -n "${_shunit_skipped_}" ] \ + && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}" + _shunit_msg_="${_shunit_msg_})" + fi + + echo + echo ${_shunit_msg_} + __shunit_reportGenerated=${SHUNIT_TRUE} + + unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_ +} + +# Test for whether a function should be skipped. +# +# Args: +# None +# Returns: +# boolean: whether the test should be skipped (TRUE/FALSE constant) +_shunit_shouldSkip() +{ + [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} + _shunit_assertSkip +} + +# Records a successful test. +# +# Args: +# None +_shunit_assertPass() +{ + __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +# Records a test failure. +# +# Args: +# message: string: failure message to provide user +_shunit_assertFail() +{ + _shunit_msg_=$1 + + __shunit_testSuccess=${SHUNIT_FALSE} + __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` + echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}" + + unset _shunit_msg_ +} + +# Records a skipped test. +# +# Args: +# None +_shunit_assertSkip() +{ + __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +# Prepare a script filename for sourcing. +# +# Args: +# script: string: path to a script to source +# Returns: +# string: filename prefixed with ./ (if necessary) +_shunit_prepForSourcing() +{ + _shunit_script_=$1 + case "${_shunit_script_}" in + /*|./*) echo "${_shunit_script_}" ;; + *) echo "./${_shunit_script_}" ;; + esac + unset _shunit_script_ +} + +# Escape a character in a string. +# +# Args: +# c: string: unescaped character +# s: string: to escape character in +# Returns: +# string: with escaped character(s) +_shunit_escapeCharInStr() +{ + [ -n "$2" ] || return # no point in doing work on an empty string + + # Note: using shorter variable names to prevent conflicts with + # _shunit_escapeCharactersInString(). + _shunit_c_=$1 + _shunit_s_=$2 + + + # escape the character + echo ''${_shunit_s_}'' |sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' + + unset _shunit_c_ _shunit_s_ +} + +# Escape a character in a string. +# +# Args: +# str: string: to escape characters in +# Returns: +# string: with escaped character(s) +_shunit_escapeCharactersInString() +{ + [ -n "$1" ] || return # no point in doing work on an empty string + + _shunit_str_=$1 + + # Note: using longer variable names to prevent conflicts with + # _shunit_escapeCharInStr(). + for _shunit_char_ in '"' '$' "'" '`'; do + _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` + done + + echo "${_shunit_str_}" + unset _shunit_char_ _shunit_str_ +} + +# Extract list of functions to run tests against. +# +# Args: +# script: string: name of script to extract functions from +# Returns: +# string: of function names +_shunit_extractTestFunctions() +{ + _shunit_script_=$1 + + # extract the lines with test function names, strip of anything besides the + # function name, and output everything on a single line. + _shunit_regex_='^[ ]*(function )*test[A-Za-z0-9_]* *\(\)' + egrep "${_shunit_regex_}" "${_shunit_script_}" \ + |sed 's/^[^A-Za-z0-9_]*//;s/^function //;s/\([A-Za-z0-9_]*\).*/\1/g' \ + |xargs + + unset _shunit_regex_ _shunit_script_ +} + +#------------------------------------------------------------------------------ +# main +# + +# determine the operating mode +if [ $# -eq 0 ]; then + __shunit_script=${__SHUNIT_PARENT} + __shunit_mode=${__SHUNIT_MODE_SOURCED} +else + __shunit_script=$1 + [ -r "${__shunit_script}" ] || \ + _shunit_fatal "unable to read from ${__shunit_script}" + __shunit_mode=${__SHUNIT_MODE_STANDALONE} +fi + +# create a temporary storage location +__shunit_tmpDir=`_shunit_mktempDir` + +# provide a public temporary directory for unit test scripts +# TODO(kward): document this +SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" +mkdir "${SHUNIT_TMPDIR}" + +# setup traps to clean up after ourselves +trap '_shunit_cleanup EXIT' 0 +trap '_shunit_cleanup INT' 2 +trap '_shunit_cleanup TERM' 15 + +# create phantom functions to work around issues with Cygwin +_shunit_mktempFunc +PATH="${__shunit_tmpDir}:${PATH}" + +# make sure phantom functions are executable. this will bite if /tmp (or the +# current $TMPDIR) points to a path on a partition that was mounted with the +# 'noexec' option. the noexec command was created with _shunit_mktempFunc(). +noexec 2>/dev/null || _shunit_fatal \ + 'please declare TMPDIR with path on partition with exec permission' + +# we must manually source the tests in standalone mode +if [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then + . "`_shunit_prepForSourcing \"${__shunit_script}\"`" +fi + +# execute the oneTimeSetUp function (if it exists) +oneTimeSetUp + +# execute the suite function defined in the parent test script +# deprecated as of 2.1.0 +suite + +# if no suite function was defined, dynamically build a list of functions +if [ -z "${__shunit_suite}" ]; then + shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` + for shunit_func_ in ${shunit_funcs_}; do + suite_addTest ${shunit_func_} + done +fi +unset shunit_func_ shunit_funcs_ + +# execute the tests +_shunit_execSuite + +# execute the oneTimeTearDown function (if it exists) +oneTimeTearDown + +# generate the report +_shunit_generateReport + +# that's it folks +[ ${__shunit_testsFailed} -eq 0 ] +exit $? diff --git a/dev/tests/shunit2/shunit2_test.sh b/dev/tests/shunit2/shunit2_test.sh new file mode 100755 index 0000000..d8f5a9c --- /dev/null +++ b/dev/tests/shunit2/shunit2_test.sh @@ -0,0 +1,124 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit test suite runner. +# +# This script runs all the unit tests that can be found, and generates a nice +# report of the tests. + +MY_NAME=`basename $0` +MY_PATH=`dirname $0` + +PREFIX='shunit2_test_' +SHELLS='/bin/sh /bin/bash /bin/dash /bin/ksh /bin/pdksh /bin/zsh' +TESTS='' +for test in ${PREFIX}[a-z]*.sh; do + TESTS="${TESTS} ${test}" +done + +# load common unit test functions +. ../lib/versions +. ./shunit2_test_helpers + +usage() +{ + echo "usage: ${MY_NAME} [-e key=val ...] [-s shell(s)] [-t test(s)]" +} + +env='' + +# process command line flags +while getopts 'e:hs:t:' opt; do + case ${opt} in + e) # set an environment variable + key=`expr "${OPTARG}" : '\([^=]*\)='` + val=`expr "${OPTARG}" : '[^=]*=\(.*\)'` + if [ -z "${key}" -o -z "${val}" ]; then + usage + exit 1 + fi + eval "${key}='${val}'" + export ${key} + env="${env:+${env} }${key}" + ;; + h) usage; exit 0 ;; # output help + s) shells=${OPTARG} ;; # list of shells to run + t) tests=${OPTARG} ;; # list of tests to run + *) usage; exit 1 ;; + esac +done +shift `expr ${OPTIND} - 1` + +# fill shells and/or tests +shells=${shells:-${SHELLS}} +tests=${tests:-${TESTS}} + +# error checking +if [ -z "${tests}" ]; then + th_error 'no tests found to run; exiting' + exit 1 +fi + +cat <&1; ) + done +done diff --git a/dev/tests/shunit2/shunit2_test_asserts.sh b/dev/tests/shunit2/shunit2_test_asserts.sh new file mode 100755 index 0000000..38647ec --- /dev/null +++ b/dev/tests/shunit2/shunit2_test_asserts.sh @@ -0,0 +1,206 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit test for assert functions + +# load test helpers +. ./shunit2_test_helpers + +#------------------------------------------------------------------------------ +# suite tests +# + +commonEqualsSame() +{ + fn=$1 + + ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'equal' $? "${stdoutF}" "${stderrF}" + + ( ${fn} "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'equal; with msg' $? "${stdoutF}" "${stderrF}" + + ( ${fn} 'abc def' 'abc def' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'equal with spaces' $? "${stdoutF}" "${stderrF}" + + ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'not equal' $? "${stdoutF}" "${stderrF}" + + ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'null values' $? "${stdoutF}" "${stderrF}" + + ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +commonNotEqualsSame() +{ + fn=$1 + + ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not same' $? "${stdoutF}" "${stderrF}" + + ( ${fn} "${MSG}" 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not same, with msg' $? "${stdoutF}" "${stderrF}" + + ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}" + + ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}" + + ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +testAssertEquals() +{ + commonEqualsSame 'assertEquals' +} + +testAssertNotEquals() +{ + commonNotEqualsSame 'assertNotEquals' +} + +testAssertSame() +{ + commonEqualsSame 'assertSame' +} + +testAssertNotSame() +{ + commonNotEqualsSame 'assertNotSame' +} + +testAssertNull() +{ + ( assertNull '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'null' $? "${stdoutF}" "${stderrF}" + + ( assertNull "${MSG}" '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'null, with msg' $? "${stdoutF}" "${stderrF}" + + ( assertNull 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'not null' $? "${stdoutF}" "${stderrF}" + + ( assertNull >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( assertNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +testAssertNotNull() +{ + ( assertNotNull 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not null' $? "${stdoutF}" "${stderrF}" + + ( assertNotNull "${MSG}" 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not null, with msg' $? "${stdoutF}" "${stderrF}" + + ( assertNotNull 'x"b' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not null, with double-quote' $? \ + "${stdoutF}" "${stderrF}" + + ( assertNotNull "x'b" >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not null, with single-quote' $? \ + "${stdoutF}" "${stderrF}" + + ( assertNotNull 'x$b' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not null, with dollar' $? \ + "${stdoutF}" "${stderrF}" + + ( assertNotNull 'x`b' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'not null, with backtick' $? \ + "${stdoutF}" "${stderrF}" + + ( assertNotNull '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}" + + # there is no test for too few arguments as $1 might actually be null + + ( assertNotNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +testAssertTrue() +{ + ( assertTrue 0 >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'true' $? "${stdoutF}" "${stderrF}" + + ( assertTrue "${MSG}" 0 >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'true, with msg' $? "${stdoutF}" "${stderrF}" + + ( assertTrue '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'true condition' $? "${stdoutF}" "${stderrF}" + + ( assertTrue 1 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'false' $? "${stdoutF}" "${stderrF}" + + ( assertTrue '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'false condition' $? "${stdoutF}" "${stderrF}" + + ( assertTrue '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}" + + ( assertTrue >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( assertTrue arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +testAssertFalse() +{ + ( assertFalse 1 >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'false' $? "${stdoutF}" "${stderrF}" + + ( assertFalse "${MSG}" 1 >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'false, with msg' $? "${stdoutF}" "${stderrF}" + + ( assertFalse '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" ) + th_assertTrueWithNoOutput 'false condition' $? "${stdoutF}" "${stderrF}" + + ( assertFalse 0 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'true' $? "${stdoutF}" "${stderrF}" + + ( assertFalse '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}" + + ( assertFalse '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}" + + ( assertFalse >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( assertFalse arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +#------------------------------------------------------------------------------ +# suite functions +# + +oneTimeSetUp() +{ + th_oneTimeSetUp + + MSG='This is a test message' +} + +# load and run shUnit2 +[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 +. ${TH_SHUNIT} diff --git a/dev/tests/shunit2/shunit2_test_failures.sh b/dev/tests/shunit2/shunit2_test_failures.sh new file mode 100755 index 0000000..933a0b1 --- /dev/null +++ b/dev/tests/shunit2/shunit2_test_failures.sh @@ -0,0 +1,86 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit test for failure functions + +# load common unit-test functions +. ./shunit2_test_helpers + +#----------------------------------------------------------------------------- +# suite tests +# + +testFail() +{ + ( fail >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'fail' $? "${stdoutF}" "${stderrF}" + + ( fail "${MSG}" >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'fail with msg' $? "${stdoutF}" "${stderrF}" + + ( fail arg1 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +testFailNotEquals() +{ + ( failNotEquals 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}" + + ( failNotEquals "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}" + + ( failNotEquals 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}" + + ( failNotEquals '' '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}" + + ( failNotEquals >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( failNotEquals arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +testFailSame() +{ + ( failSame 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}" + + ( failSame "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}" + + ( failSame 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}" + + ( failSame '' '' >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}" + + ( failSame >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" + + ( failSame arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) + th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" +} + +#----------------------------------------------------------------------------- +# suite functions +# + +oneTimeSetUp() +{ + th_oneTimeSetUp + + MSG='This is a test message' +} + +# load and run shUnit2 +[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 +. ${TH_SHUNIT} diff --git a/dev/tests/shunit2/shunit2_test_helpers b/dev/tests/shunit2/shunit2_test_helpers new file mode 100644 index 0000000..a9989d6 --- /dev/null +++ b/dev/tests/shunit2/shunit2_test_helpers @@ -0,0 +1,229 @@ +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit test common functions + +# treat unset variables as an error when performing parameter expansion +set -u + +# set shwordsplit for zsh +[ -n "${ZSH_VERSION:-}" ] && setopt shwordsplit + +# +# constants +# + +# path to shUnit2 library. can be overridden by setting SHUNIT_INC +TH_SHUNIT=${SHUNIT_INC:-./shunit2} + +# configure debugging. set the DEBUG environment variable to any +# non-empty value to enable debug output, or TRACE to enable trace +# output. +TRACE=${TRACE:+'th_trace '} +[ -n "${TRACE}" ] && DEBUG=1 +[ -z "${TRACE}" ] && TRACE=':' + +DEBUG=${DEBUG:+'th_debug '} +[ -z "${DEBUG}" ] && DEBUG=':' + +# +# variables +# + +th_RANDOM=0 + +# +# functions +# + +# message functions +th_trace() { echo "${MY_NAME}:TRACE $@" >&2; } +th_debug() { echo "${MY_NAME}:DEBUG $@" >&2; } +th_info() { echo "${MY_NAME}:INFO $@" >&2; } +th_warn() { echo "${MY_NAME}:WARN $@" >&2; } +th_error() { echo "${MY_NAME}:ERROR $@" >&2; } +th_fatal() { echo "${MY_NAME}:FATAL $@" >&2; } + +# output subtest name +th_subtest() { echo " $@" >&2; } + +th_oneTimeSetUp() +{ + # these files will be cleaned up automatically by shUnit2 + stdoutF="${SHUNIT_TMPDIR}/stdout" + stderrF="${SHUNIT_TMPDIR}/stderr" + returnF="${SHUNIT_TMPDIR}/return" + expectedF="${SHUNIT_TMPDIR}/expected" +} + +# generate a random number +th_generateRandom() +{ + tfgr_random=${th_RANDOM} + + while [ "${tfgr_random}" = "${th_RANDOM}" ]; do + if [ -n "${RANDOM:-}" ]; then + # $RANDOM works + tfgr_random=${RANDOM}${RANDOM}${RANDOM}$$ + elif [ -r '/dev/urandom' ]; then + tfgr_random=`od -vAn -N4 -tu4 >> STDOUT' >&2 + cat "${_th_stdout_}" >&2 + fi + if [ -n "${_th_stderr_}" -a -s "${_th_stderr_}" ]; then + echo '>>> STDERR' >&2 + cat "${_th_stderr_}" >&2 + fi + if [ -n "${_th_stdout_}" -o -n "${_th_stderr_}" ]; then + echo '<<< end output' >&2 + fi + fi + + unset _th_return_ _th_stdout_ _th_stderr_ +} + +# +# main +# + +${TRACE} 'trace output enabled' +${DEBUG} 'debug output enabled' diff --git a/dev/tests/shunit2/shunit2_test_macros.sh b/dev/tests/shunit2/shunit2_test_macros.sh new file mode 100755 index 0000000..ce57b14 --- /dev/null +++ b/dev/tests/shunit2/shunit2_test_macros.sh @@ -0,0 +1,246 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit test for macros. + +# load test helpers +. ./shunit2_test_helpers + +#------------------------------------------------------------------------------ +# suite tests +# + +testAssertEquals() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_EQUALS_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_EQUALS_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testAssertNotEquals() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_NOT_EQUALS_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NOT_EQUALS_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_NOT_EQUALS_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NOT_EQUALS_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testSame() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_SAME_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_SAME_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testNotSame() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_NOT_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NOT_SAME_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_NOT_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NOT_SAME_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testNull() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_NULL_} 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NULL_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_NULL_} '"some msg"' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NULL_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testNotNull() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_NOT_NULL_} '' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NOT_NULL_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_NOT_NULL_} '"some msg"' '""' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_NOT_NULL_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stdoutF}" "${stderrF}" >&2 +} + +testAssertTrue() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_TRUE_} ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_TRUE_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + + ( ${_ASSERT_TRUE_} '"some msg"' ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_TRUE_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testAssertFalse() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_ASSERT_FALSE_} ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_FALSE_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_ASSERT_FALSE_} '"some msg"' ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_ASSERT_FALSE_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testFail() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_FAIL_} >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_FAIL_} '"some msg"' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testFailNotEquals() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_FAIL_NOT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_NOT_EQUALS_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_FAIL_NOT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_NOT_EQUALS_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testFailSame() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_FAIL_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_SAME_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_FAIL_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_SAME_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testFailNotSame() +{ + # start skipping if LINENO not available + [ -z "${LINENO:-}" ] && startSkipping + + ( ${_FAIL_NOT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_NOT_SAME_ failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 + + ( ${_FAIL_NOT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) + grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null + rtrn=$? + assertTrue '_FAIL_NOT_SAME_ w/ msg failure' ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +#------------------------------------------------------------------------------ +# suite functions +# + +oneTimeSetUp() +{ + th_oneTimeSetUp +} + +# load and run shUnit2 +[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 +. ${TH_SHUNIT} diff --git a/dev/tests/shunit2/shunit2_test_misc.sh b/dev/tests/shunit2/shunit2_test_misc.sh new file mode 100755 index 0000000..d264628 --- /dev/null +++ b/dev/tests/shunit2/shunit2_test_misc.sh @@ -0,0 +1,160 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit tests of miscellaneous things + +# load test helpers +. ./shunit2_test_helpers + +#------------------------------------------------------------------------------ +# suite tests +# + +# Note: the test script is prefixed with '#' chars so that shUnit2 does not +# incorrectly interpret the embedded functions as real functions. +testUnboundVariable() +{ + unittestF="${SHUNIT_TMPDIR}/unittest" + sed 's/^#//' >"${unittestF}" <"${stdoutF}" 2>"${stderrF}" ) + assertFalse 'expected a non-zero exit value' $? + grep '^ASSERT:Unknown failure' "${stdoutF}" >/dev/null + assertTrue 'assert message was not generated' $? + grep '^Ran [0-9]* test' "${stdoutF}" >/dev/null + assertTrue 'test count message was not generated' $? + grep '^FAILED' "${stdoutF}" >/dev/null + assertTrue 'failure message was not generated' $? +} + +testIssue7() +{ + ( assertEquals 'Some message.' 1 2 >"${stdoutF}" 2>"${stderrF}" ) + diff "${stdoutF}" - >/dev/null < but was:<2> +EOF + rtrn=$? + assertEquals ${SHUNIT_TRUE} ${rtrn} + [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 +} + +testPrepForSourcing() +{ + assertEquals '/abc' `_shunit_prepForSourcing '/abc'` + assertEquals './abc' `_shunit_prepForSourcing './abc'` + assertEquals './abc' `_shunit_prepForSourcing 'abc'` +} + +testEscapeCharInStr() +{ + actual=`_shunit_escapeCharInStr '\' ''` + assertEquals '' "${actual}" + assertEquals 'abc\\' `_shunit_escapeCharInStr '\' 'abc\'` + assertEquals 'abc\\def' `_shunit_escapeCharInStr '\' 'abc\def'` + assertEquals '\\def' `_shunit_escapeCharInStr '\' '\def'` + + actual=`_shunit_escapeCharInStr '"' ''` + assertEquals '' "${actual}" + assertEquals 'abc\"' `_shunit_escapeCharInStr '"' 'abc"'` + assertEquals 'abc\"def' `_shunit_escapeCharInStr '"' 'abc"def'` + assertEquals '\"def' `_shunit_escapeCharInStr '"' '"def'` + + actual=`_shunit_escapeCharInStr '$' ''` + assertEquals '' "${actual}" + assertEquals 'abc\$' `_shunit_escapeCharInStr '$' 'abc$'` + assertEquals 'abc\$def' `_shunit_escapeCharInStr '$' 'abc$def'` + assertEquals '\$def' `_shunit_escapeCharInStr '$' '$def'` + +# actual=`_shunit_escapeCharInStr "'" ''` +# assertEquals '' "${actual}" +# assertEquals "abc\\'" `_shunit_escapeCharInStr "'" "abc'"` +# assertEquals "abc\\'def" `_shunit_escapeCharInStr "'" "abc'def"` +# assertEquals "\\'def" `_shunit_escapeCharInStr "'" "'def"` + +# # must put the backtick in a variable so the shell doesn't misinterpret it +# # while inside a backticked sequence (e.g. `echo '`'` would fail). +# backtick='`' +# actual=`_shunit_escapeCharInStr ${backtick} ''` +# assertEquals '' "${actual}" +# assertEquals '\`abc' \ +# `_shunit_escapeCharInStr "${backtick}" ${backtick}'abc'` +# assertEquals 'abc\`' \ +# `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}` +# assertEquals 'abc\`def' \ +# `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}'def'` +} + +testEscapeCharInStr_specialChars() +{ + # make sure our forward slash doesn't upset sed + assertEquals '/' `_shunit_escapeCharInStr '\' '/'` + + # some shells escape these differently + #assertEquals '\\a' `_shunit_escapeCharInStr '\' '\a'` + #assertEquals '\\b' `_shunit_escapeCharInStr '\' '\b'` +} + +# Test the various ways of declaring functions. +# +# Prefixing (then stripping) with comment symbol so these functions aren't +# treated as real functions by shUnit2. +testExtractTestFunctions() +{ + f="${SHUNIT_TMPDIR}/extract_test_functions" + sed 's/^#//' <"${f}" +#testABC() { echo 'ABC'; } +#test_def() { +# echo 'def' +#} +#testG3 () +#{ +# echo 'G3' +#} +#function test4() { echo '4'; } +# test5() { echo '5'; } +#some_test_function() { echo 'some func'; } +#func_with_test_vars() { +# testVariable=1234 +#} +EOF + + actual=`_shunit_extractTestFunctions "${f}"` + assertEquals 'testABC test_def testG3 test4 test5' "${actual}" +} + +#------------------------------------------------------------------------------ +# suite functions +# + +setUp() +{ + for f in ${expectedF} ${stdoutF} ${stderrF}; do + cp /dev/null ${f} + done +} + +oneTimeSetUp() +{ + th_oneTimeSetUp +} + +# load and run shUnit2 +[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 +. ${TH_SHUNIT} diff --git a/dev/tests/shunit2/shunit2_test_standalone.sh b/dev/tests/shunit2/shunit2_test_standalone.sh new file mode 100755 index 0000000..2ac4725 --- /dev/null +++ b/dev/tests/shunit2/shunit2_test_standalone.sh @@ -0,0 +1,41 @@ +#! /bin/sh +# $Id$ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2010 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 unit test for standalone operation. +# +# This unit test is purely to test that calling shunit2 directly, while passing +# the name of a unit test script, works. When run, this script determines if it +# is running as a standalone program, and calls main() if it is. + +ARGV0=`basename "$0"` + +# load test helpers +. ./shunit2_test_helpers + +#------------------------------------------------------------------------------ +# suite tests +# + +testStandalone() +{ + assertTrue ${SHUNIT_TRUE} +} + +#------------------------------------------------------------------------------ +# main +# + +main() +{ + ${TH_SHUNIT} "${ARGV0}" +} + +# are we running as a standalone? +if [ "${ARGV0}" = 'shunit2_test_standalone.sh' ]; then + if [ $# -gt 0 ]; then main "$@"; else main; fi +fi From 8393c2e7cdf9ae3b90ad49f1edd191f5d1878c2d Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 10:27:08 +0200 Subject: [PATCH 10/16] Updated changelog --- CHANGELOG.md | 12 +- README.md | 3 + tests/conf/local.conf | 143 --- tests/conf/remote.conf | 143 --- tests/run.sh | 345 ------- tests/shunit2/shunit2 | 1067 ---------------------- tests/shunit2/shunit2_test.sh | 124 --- tests/shunit2/shunit2_test_asserts.sh | 206 ----- tests/shunit2/shunit2_test_failures.sh | 86 -- tests/shunit2/shunit2_test_helpers | 229 ----- tests/shunit2/shunit2_test_macros.sh | 246 ----- tests/shunit2/shunit2_test_misc.sh | 160 ---- tests/shunit2/shunit2_test_standalone.sh | 41 - 13 files changed, 14 insertions(+), 2791 deletions(-) delete mode 100755 tests/conf/local.conf delete mode 100755 tests/conf/remote.conf delete mode 100755 tests/run.sh delete mode 100755 tests/shunit2/shunit2 delete mode 100755 tests/shunit2/shunit2_test.sh delete mode 100755 tests/shunit2/shunit2_test_asserts.sh delete mode 100755 tests/shunit2/shunit2_test_failures.sh delete mode 100644 tests/shunit2/shunit2_test_helpers delete mode 100755 tests/shunit2/shunit2_test_macros.sh delete mode 100755 tests/shunit2/shunit2_test_misc.sh delete mode 100755 tests/shunit2/shunit2_test_standalone.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a63ad7f..7b8473c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,22 @@ KNOWN ISSUES RECENT CHANGES -------------- + +- Made unix signals posix compliant +- Config file upgrade script now updates header +! test if waitfortaskcompletion kill self works okay with osync +- Improved batch runner +- Made keep logging value configurable and not mandatory +- Fixed handling of processes in uninterruptible sleep state ! update doc on sudoers paths - Parallelized sync functions - #TODO: explain bandwidth parameter, and CONFLICT_PREVALANCE option + - Rewrite sync resume process + ! Remove conflict prevalance + - !doc about bandwidth - Added options to ignore permissions, ownership and groups - Refactored WaitFor... functions into one - Improved execution speed + - Rewrite sync resume process - Added parallel execution for most secondary fuctions - Lowered sleep time in wait functions - Removed trivial sleep and forking in remote deletion code, send the whole function to background instead diff --git a/README.md b/README.md index 421c86e..1871e33 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ Osync provides the following capabilities - Batch runner for multiple sync tasks with rerun option for failed sync tasks - ACL synchronization +osync is a state synchronizer. This means that it doesn't have to monitor files for changes. Instead, it compares replica lists between runs. +A full run takes about 2 seconds on a local-local replication and about 10 seconds on a local-remote replication. +Disabling some features file like attributes preservation and disk space checks may speed up execution. osync uses a initiator / target sync schema. It can sync local to local or local to remote directories. By definition, initiator replica is always a local directory on the system osync runs on. osync uses pidlocks to prevent multiple concurrent sync processes on/to the same initiator / target replica. You may launch concurrent sync processes on the same system but only for different initiator replicas. diff --git a/tests/conf/local.conf b/tests/conf/local.conf deleted file mode 100755 index b52d0bf..0000000 --- a/tests/conf/local.conf +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -###### Osync - Rsync based two way sync engine with fault tolerance -###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2611201401 - -## ---------- GENERAL OPTIONS - -## Sync job identification -SYNC_ID="sync_test" - -## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one. -MASTER_SYNC_DIR="master" -SLAVE_SYNC_DIR="slave" -#SLAVE_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir2" -## If slave replica is a remote directory, you must specify a RSA key (please use full path). Please see documentation for further information. -#SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa" - -## Create sync directories if they do not exist -CREATE_DIRS=no - -## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist) -LOGFILE="" - - -## List of directories to exclude from sync on both sides (rsync patterns, wildcards work). -## Paths are relative to sync dirs. List elements are separated by a semicolon. -RSYNC_EXCLUDE_PATTERN="" -#RSYNC_EXCLUDE_PATTERN="tmp;archives" - -## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. -## This file has to be in the same directory as the config file -## Paths are relative to sync dirs. One element per line. -RSYNC_EXCLUDE_FROM="" -#RSYNC_EXCLUDE_FROM="exclude.list" - -## List elements separator char. You may set an alternative separator char for your directories lists above. -PATH_SEPARATOR_CHAR=";" - -## Generate an alert if master or slave replicas have less free space than given value in KB. -MINIMUM_SPACE=10240 - -## Bandwidth limit Kbytes / second. Leave 0 to disable limitation -BANDWIDTH=0 - -## If enabled, synchronization on remote system will be processed as superuser. See documentation for /etc/sudoers file configuration. -SUDO_EXEC=no -## Paranoia option. Don't change this unless you read the documentation. -RSYNC_EXECUTABLE=rsync - -## ---------- REMOTE SYNC OPTIONS - -## ssh compression should be used unless your remote connection is good enough (LAN) -SSH_COMPRESSION=yes - -## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync. -REMOTE_HOST_PING=no - -## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync. -## If you use this function, you should set more than one 3rd party host, and be sure you can ping them. -## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute. -REMOTE_3RD_PARTY_HOSTS="" - -## Remote rsync executable path. Leave this empty in most cases -RSYNC_REMOTE_PATH="" - -## ---------- MISC OPTIONS - -## Preserve ACLS. Make sure source and target FS can manage same ACLs or you'll get loads of errors. -PRESERVE_ACL=no -## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors. -PRESERVE_XATTR=no -## Transforms symlinks into referent files/dirs -COPY_SYMLINKS=no -## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root. -KEEP_DIRLINKS=no -## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them. -PRESERVE_HARDLINKS=no - -## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression. -RSYNC_COMPRESS=yes - -## Maximum execution time (in seconds) for sync process. Soft exec time only generates a warning. Hard exec time will generate a warning and stop sync process. -SOFT_MAX_EXEC_TIME=7200 -HARD_MAX_EXEC_TIME=10600 - -## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish. -MIN_WAIT=60 - -## ---------- BACKUP AND DELETION OPTIONS - -## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups -CONFLICT_BACKUP=yes -## Keep multiple backup versions of the same file. Warning, This can be very space consuming. -CONFLICT_BACKUP_MULTIPLE=no -## Osync will clean backup files after a given number of days. Setting this to 0 will disable cleaning and keep backups forever. Warning: This can be very space consuming. -CONFLICT_BACKUP_DAYS=30 -## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica. -CONFLICT_PREVALANCE=master - -## On deletion propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted -SOFT_DELETE=yes -## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming. -SOFT_DELETE_DAYS=30 - -## ---------- RESUME OPTIONS - -## Try to resume an aborted sync task -RESUME_SYNC=yes -## Number maximum resume tries before initiating a fresh sync. -RESUME_TRY=2 -## When a pidlock exists on slave replica that does not correspond to master's sync-id, force pidlock removal. Be careful with this option if you have multiple masters. -FORCE_STRANGER_LOCK_RESUME=no - -## Keep partial uploads that can be resumed on next run, experimental feature -PARTIAL=no - -## ---------- ALERT OPTIONS - -## List of alert mails separated by spaces -DESTINATION_MAILS="your@alert.tld" - -## Windows (MSYS environment) only mail options (used with sendemail.exe from Brandon Zehm) -SENDER_MAIL="alert@your.system.tld" -SMTP_SERVER=smtp.your.isp.tld -SMTP_USER= -SMTP_PASSWORD= - -## ---------- EXECUTION HOOKS - -## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_SYNC is set). -LOCAL_RUN_BEFORE_CMD="" -LOCAL_RUN_AFTER_CMD="" - -REMOTE_RUN_BEFORE_CMD="" -REMOTE_RUN_AFTER_CMD="" - -## Max execution time of commands before they get force killed. Leave 0 if you don't wan't this to happen. Time is specified in seconds. -MAX_EXEC_TIME_PER_CMD_BEFORE=0 -MAX_EXEC_TIME_PER_CMD_AFTER=0 - -## Stops Osync execution if one of the above commands fail -STOP_ON_CMD_ERROR=yes diff --git a/tests/conf/remote.conf b/tests/conf/remote.conf deleted file mode 100755 index 475e0c9..0000000 --- a/tests/conf/remote.conf +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -###### Osync - Rsync based two way sync engine with fault tolerance -###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2611201401 - -## ---------- GENERAL OPTIONS - -## Sync job identification -SYNC_ID="sync_test" - -## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one. -MASTER_SYNC_DIR="master" -SLAVE_SYNC_DIR="ssh://localhost//tmp/osync_tests/remote/slave" -#SLAVE_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir2" -## If slave replica is a remote directory, you must specify a RSA key (please use full path). Please see documentation for further information. -#SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa" - -## Create sync directories if they do not exist -CREATE_DIRS=yes - -## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist) -LOGFILE="" - - -## List of directories to exclude from sync on both sides (rsync patterns, wildcards work). -## Paths are relative to sync dirs. List elements are separated by a semicolon. -RSYNC_EXCLUDE_PATTERN="" -#RSYNC_EXCLUDE_PATTERN="tmp;archives" - -## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. -## This file has to be in the same directory as the config file -## Paths are relative to sync dirs. One element per line. -RSYNC_EXCLUDE_FROM="" -#RSYNC_EXCLUDE_FROM="exclude.list" - -## List elements separator char. You may set an alternative separator char for your directories lists above. -PATH_SEPARATOR_CHAR=";" - -## Generate an alert if master or slave replicas have less free space than given value in KB. -MINIMUM_SPACE=10240 - -## Bandwidth limit Kbytes / second. Leave 0 to disable limitation -BANDWIDTH=0 - -## If enabled, synchronization on remote system will be processed as superuser. See documentation for /etc/sudoers file configuration. -SUDO_EXEC=no -## Paranoia option. Don't change this unless you read the documentation. -RSYNC_EXECUTABLE=rsync - -## ---------- REMOTE SYNC OPTIONS - -## ssh compression should be used unless your remote connection is good enough (LAN) -SSH_COMPRESSION=yes - -## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync. -REMOTE_HOST_PING=no - -## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync. -## If you use this function, you should set more than one 3rd party host, and be sure you can ping them. -## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute. -REMOTE_3RD_PARTY_HOSTS="" - -## Remote rsync executable path. Leave this empty in most cases -RSYNC_REMOTE_PATH="" - -## ---------- MISC OPTIONS - -## Preserve ACLS. Make sure source and target FS can manage same ACLs or you'll get loads of errors. -PRESERVE_ACL=no -## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors. -PRESERVE_XATTR=no -## Transforms symlinks into referent files/dirs -COPY_SYMLINKS=no -## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root. -KEEP_DIRLINKS=no -## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them. -PRESERVE_HARDLINKS=no - -## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression. -RSYNC_COMPRESS=yes - -## Maximum execution time (in seconds) for sync process. Soft exec time only generates a warning. Hard exec time will generate a warning and stop sync process. -SOFT_MAX_EXEC_TIME=7200 -HARD_MAX_EXEC_TIME=10600 - -## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish. -MIN_WAIT=60 - -## ---------- BACKUP AND DELETION OPTIONS - -## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups -CONFLICT_BACKUP=yes -## Keep multiple backup versions of the same file. Warning, This can be very space consuming. -CONFLICT_BACKUP_MULTIPLE=no -## Osync will clean backup files after a given number of days. Setting this to 0 will disable cleaning and keep backups forever. Warning: This can be very space consuming. -CONFLICT_BACKUP_DAYS=30 -## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica. -CONFLICT_PREVALANCE=master - -## On deletion propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted -SOFT_DELETE=yes -## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming. -SOFT_DELETE_DAYS=30 - -## ---------- RESUME OPTIONS - -## Try to resume an aborted sync task -RESUME_SYNC=yes -## Number maximum resume tries before initiating a fresh sync. -RESUME_TRY=2 -## When a pidlock exists on slave replica that does not correspond to master's sync-id, force pidlock removal. Be careful with this option if you have multiple masters. -FORCE_STRANGER_LOCK_RESUME=no - -## Keep partial uploads that can be resumed on next run, experimental feature -PARTIAL=no - -## ---------- ALERT OPTIONS - -## List of alert mails separated by spaces -DESTINATION_MAILS="your@alert.tld" - -## Windows (MSYS environment) only mail options (used with sendemail.exe from Brandon Zehm) -SENDER_MAIL="alert@your.system.tld" -SMTP_SERVER=smtp.your.isp.tld -SMTP_USER= -SMTP_PASSWORD= - -## ---------- EXECUTION HOOKS - -## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_SYNC is set). -LOCAL_RUN_BEFORE_CMD="" -LOCAL_RUN_AFTER_CMD="" - -REMOTE_RUN_BEFORE_CMD="" -REMOTE_RUN_AFTER_CMD="" - -## Max execution time of commands before they get force killed. Leave 0 if you don't wan't this to happen. Time is specified in seconds. -MAX_EXEC_TIME_PER_CMD_BEFORE=0 -MAX_EXEC_TIME_PER_CMD_AFTER=0 - -## Stops Osync execution if one of the above commands fail -STOP_ON_CMD_ERROR=yes diff --git a/tests/run.sh b/tests/run.sh deleted file mode 100755 index f3dfe3d..0000000 --- a/tests/run.sh +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env bash - -# Test dir -TMP="/tmp/osync_tests" -# SSH port used for remote tests -SSH_PORT=49999 - -# Get dir the tests are stored in -TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -cd "$TEST_DIR" - -OSYNC_EXECUTABLE="$(dirname $TEST_DIR)//osync.sh" -declare -A sandbox_osync -#sandbox_osync[quickLocal]="--master=master --slave=slave" -#sandbox_osync[quickRemote]="--master=master --slave=ssh://localhost//tmp/osync_tests/quickRemote/slave" -#sandbox_osync[local]="conf/local.conf" -#sandbox_osync[remote]="conf/remote.conf" - -oneTimeSetUp() -{ - for i in "${!sandbox_osync[@]}" - do - prepareSandbox "$i" - done -} - -oneTimeTearDown() -{ - rm -rf "$TMP" -} - -prepareSandbox() -{ - rm -rf "$TMP/$1" - mkdir -p "$TMP/$1" - pushd "$TMP/$1" >/dev/null - mkdir master - mkdir slave - mkdir expected - popd >/dev/null -} - -compareSandbox() -{ - diff -aurx .osync_workdir master slave - assertEquals 0 $? - - diff -aurx .osync_workdir master expected - assertEquals 0 $? - - diff -aurx .osync_workdir slave expected - assertEquals 0 $? -} - -syncSandbox() -{ - $OSYNC_EXECUTABLE ${sandbox_osync[$1]} >/dev/null - assertEquals 0 $? -} - -runSandbox() -{ - syncSandbox "$1" - compareSandbox -} - -joinSandbox() -{ - cd "$TMP/$1" -} - -### Tests ### -# One empty file -_testOneEmptyFile() -{ - joinSandbox "$1" - - # Add one empty file - touch "$2/testOneEmpty" - touch expected/testOneEmpty - runSandbox "$1" - - # Change one empty file - echo "Test" > "$2/testOneEmpty" - cp "$2/testOneEmpty" expected/testOneEmpty - runSandbox "$1" - - # Empty one file - echo -n "" > "$2/testOneEmpty" - cp "$2/testOneEmpty" expected/testOneEmpty - runSandbox "$1" - - # Delete one empty file - cp "$2/testOneEmpty" testOneEmpty - rm "$2/testOneEmpty" - rm expected/testOneEmpty - runSandbox "$1" - # Backup check - if [ "$2" == "master" ] - then - diff -aur slave/.osync_workdir/deleted/testOneEmpty testOneEmpty - else - diff -aur master/.osync_workdir/deleted/testOneEmpty testOneEmpty - fi - assertEquals 0 $? -} - -testQuickLocalMasterOneEmptyFile() -{ - _testOneEmptyFile quickLocal master -} - -testQuickLocalSlaveOneEmptyFile() -{ - _testOneEmptyFile quickLocal slave -} - -testQuickRemoteMasterOneEmptyFile() -{ - _testOneEmptyFile quickRemote master -} - -testQuickRemoteSlaveOneEmptyFile() -{ - _testOneEmptyFile quickRemote slave -} - -testLocalMasterOneEmptyFile() -{ - _testOneEmptyFile local master -} - -testLocalSlaveOneEmptyFile() -{ - _testOneEmptyFile local slave -} - -testRemoteMasterOneEmptyFile() -{ - _testOneEmptyFile remote master -} - -testRemoteSlaveOneEmptyFile() -{ - _testOneEmptyFile remote slave -} - -# One file -_testOneFile() -{ - joinSandbox "$1" - - # Add one file - echo "Test" > "$2/testOne" - cp "$2/testOne" expected/testOne - runSandbox "$1" - - # Change one file - echo "Test2" > "$2/testOne" - cp "$2/testOne" expected/testOne - runSandbox "$1" - - # Delete one file - cp "$2/testOne" testOne - rm "$2/testOne" - rm expected/testOne - runSandbox "$1" - # Backup check - if [ "$2" == "master" ] - then - diff -aur slave/.osync_workdir/deleted/testOne testOne - else - diff -aur master/.osync_workdir/deleted/testOne testOne - fi - assertEquals 0 $? -} - -testQuickLocalMasterOneFile() -{ - _testOneFile quickLocal master -} - -testQuickLocalSlaveOneFile() -{ - _testOneFile quickLocal slave -} - -testQuickRemoteMasterOneFile() -{ - _testOneFile quickRemote master -} - -testQuickRemoteSlaveOneFile() -{ - _testOneFile quickRemote slave -} - -testLocalMasterOneFile() -{ - _testOneFile local master -} - -testLocalSlaveOneFile() -{ - _testOneFile local slave -} - -testRemoteMasterOneFile() -{ - _testOneFile remote master -} - -testRemoteSlaveOneFile() -{ - _testOneFile remote slave -} - -# Distinct -_testDistinct() -{ - joinSandbox "$1" - - # Generate files in master - for i in testDistinctM1 testDistinctM2 testDistinctM3 - do - mkdir "master/$i" - mkdir "expected/$i" - for j in m1 m2 m3 ; do - echo "$i/$j" > "master/$i/$j" - cp "master/$i/$j" "expected/$i/$j" - done - done - - # Generate files in slave - for i in testDistinctS1 testDistinctS2 testDistinctS3 - do - mkdir "slave/$i" - mkdir "expected/$i" - for j in s1 s2 s3 ; do - echo "$i/$j" > "slave/$i/$j" - cp "slave/$i/$j" "expected/$i/$j" - done - done - - # Generate files in same directories for master and slave - for i in testDistinctMS1 testDistinctMS2 testDistinctMS3 - do - mkdir "master/$i" - mkdir "slave/$i" - mkdir "expected/$i" - for j in ms1 ms2 ms3 ; do - echo "$i/$j" > "master/$i/m-$j" - cp "master/$i/m-$j" "expected/$i/m-$j" - echo "$i/$j" > "slave/$i/s-$j" - cp "slave/$i/s-$j" "expected/$i/s-$j" - done - done - - runSandbox "$1" -} - -testQuickLocalDistinct() -{ - _testDistinct quickLocal -} - -testQuickRemoteDistinct() -{ - _testDistinct quickRemote -} - -testLocalDistinct() -{ - _testDistinct local -} - -testRemoteDistinct() -{ - _testDistinct remote -} - -# Collision -_testCollision() -{ - joinSandbox "$1" - - # Slave precedence - echo "Test1" > master/testCollision1 - echo "Test2" > slave/testCollision1 - touch -d "2004-02-29 16:21:41" master/testCollision1 - touch -d "2004-02-29 16:21:42" slave/testCollision1 - cp slave/testCollision1 expected/testCollision1 - cp master/testCollision1 testCollision1 - runSandbox "$1" - # Backup check - diff -aur master/.osync_workdir/backups/testCollision1 testCollision1 - assertEquals 0 $? - - # Master precedence - echo "Test1" > master/testCollision2 - echo "Test2" > slave/testCollision2 - touch -d "2004-02-29 16:21:42" master/testCollision2 - touch -d "2004-02-29 16:21:41" slave/testCollision2 - cp master/testCollision2 expected/testCollision2 - cp slave/testCollision2 testCollision2 - runSandbox "$1" - # Backup check - diff -aur slave/.osync_workdir/backups/testCollision2 testCollision2 - assertEquals 0 $? - - # ?? -# echo "Test1" > master/testCollision3 -# echo "Test2" > slave/testCollision3 -# touch -d "2004-02-29 16:21:42" master/testCollision3 -# touch -d "2004-02-29 16:21:42" slave/testCollision3 -# cp slave/testCollision3 expected/testCollision3 -# runSandbox "$1" -} - -testQuickLocalCollision() -{ - _testCollision quickLocal -} - -testQuickRemoteCollision() -{ - _testCollision quickRemote -} - -testLocalCollision() -{ - _testCollision local -} - -testRemoteCollision() -{ - _testCollision remote -} - -#suite() -#{ -# suite_addTest "testQuickRemoteMasterOneEmptyFile" -#} - -. shunit2/shunit2 diff --git a/tests/shunit2/shunit2 b/tests/shunit2/shunit2 deleted file mode 100755 index d6e7503..0000000 --- a/tests/shunit2/shunit2 +++ /dev/null @@ -1,1067 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# shUnit2 -- Unit testing framework for Unix shell scripts. -# http://code.google.com/p/shunit2/ -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is -# based on the popular JUnit unit testing framework for Java. - -# return if shunit already loaded -[ -n "${SHUNIT_VERSION:-}" ] && exit 0 -SHUNIT_VERSION='2.1.7pre' - -# return values that scripts can use -SHUNIT_TRUE=0 -SHUNIT_FALSE=1 -SHUNIT_ERROR=2 - -# logging functions -_shunit_warn() { echo "shunit2:WARN $@" >&2; } -_shunit_error() { echo "shunit2:ERROR $@" >&2; } -_shunit_fatal() { echo "shunit2:FATAL $@" >&2; exit ${SHUNIT_ERROR}; } - -# determine some reasonable command defaults -__SHUNIT_UNAME_S=`uname -s` -case "${__SHUNIT_UNAME_S}" in - BSD) __SHUNIT_EXPR_CMD='gexpr' ;; - *) __SHUNIT_EXPR_CMD='expr' ;; -esac - -# commands a user can override if needed -SHUNIT_EXPR_CMD=${SHUNIT_EXPR_CMD:-${__SHUNIT_EXPR_CMD}} - -# enable strict mode by default -SHUNIT_STRICT=${SHUNIT_STRICT:-${SHUNIT_TRUE}} - -# specific shell checks -if [ -n "${ZSH_VERSION:-}" ]; then - setopt |grep "^shwordsplit$" >/dev/null - if [ $? -ne ${SHUNIT_TRUE} ]; then - _shunit_fatal 'zsh shwordsplit option is required for proper operation' - fi - if [ -z "${SHUNIT_PARENT:-}" ]; then - _shunit_fatal "zsh does not pass \$0 through properly. please declare \ -\"SHUNIT_PARENT=\$0\" before calling shUnit2" - fi -fi - -# -# constants -# - -__SHUNIT_ASSERT_MSG_PREFIX='ASSERT:' -__SHUNIT_MODE_SOURCED='sourced' -__SHUNIT_MODE_STANDALONE='standalone' -__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} - -# set the constants readonly -__shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1` -echo "${__shunit_constants}" |grep '^Binary file' >/dev/null && \ - __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1` -for __shunit_const in ${__shunit_constants}; do - if [ -z "${ZSH_VERSION:-}" ]; then - readonly ${__shunit_const} - else - case ${ZSH_VERSION} in - [123].*) readonly ${__shunit_const} ;; - *) readonly -g ${__shunit_const} # declare readonly constants globally - esac - fi -done -unset __shunit_const __shunit_constants - -# -# internal variables -# - -# variables -__shunit_lineno='' # line number of executed test -__shunit_mode=${__SHUNIT_MODE_SOURCED} # operating mode -__shunit_reportGenerated=${SHUNIT_FALSE} # is report generated -__shunit_script='' # filename of unittest script (standalone mode) -__shunit_skip=${SHUNIT_FALSE} # is skipping enabled -__shunit_suite='' # suite of tests to execute - -# counts of tests -__shunit_testSuccess=${SHUNIT_TRUE} -__shunit_testsTotal=0 -__shunit_testsPassed=0 -__shunit_testsFailed=0 - -# counts of asserts -__shunit_assertsTotal=0 -__shunit_assertsPassed=0 -__shunit_assertsFailed=0 -__shunit_assertsSkipped=0 - -# macros -_SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' - -#----------------------------------------------------------------------------- -# private functions - -#----------------------------------------------------------------------------- -# assert functions -# - -# Assert that two values are equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertEquals() requires two or three arguments; $# given" - _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}${4:+ 4: $4}" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then - _shunit_assertPass - else - failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' - -# Assert that two values are not equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotEquals() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if [ "${shunit_expected_}" != "${shunit_actual_}" ]; then - _shunit_assertPass - else - failSame "${shunit_message_}" "$@" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' - -# Assert that a value is null (i.e. an empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNull() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertNull() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertTrue "${shunit_message_}" "[ -z '$1' ]" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' - -# Assert that a value is not null (i.e. a non-empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotNull() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null - _shunit_error "assertNotNull() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` - test -n "${shunit_actual_}" - assertTrue "${shunit_message_}" $? - shunit_return=$? - - unset shunit_actual_ shunit_message_ - return ${shunit_return} -} -_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' - -# Assert that two values are the same (i.e. equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' - -# Assert that two values are not the same (i.e. not equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_:-}$1" - shift - fi - assertNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is true. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertTrue 0 -# assertTrue "[ 34 -gt 23 ]" -# The folloing test will fail with a message: -# assertTrue 123 -# assertTrue "test failed" "[ -r '/non/existant/file' ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertTrue() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertTrue() takes one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # see if condition is an integer, i.e. a return value - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if [ -z "${shunit_condition_}" ]; then - # null condition - shunit_return=${SHUNIT_FALSE} - elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # possible return value. treating 0 as true, and non-zero as false. - [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE} - else - # (hopefully) a condition - ( eval ${shunit_condition_} ) >/dev/null 2>&1 - [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # record the test - if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is false. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertFalse 1 -# assertFalse "[ 'apples' = 'oranges' ]" -# The folloing test will fail with a message: -# assertFalse 0 -# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertFalse() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertFalse() quires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # see if condition is an integer, i.e. a return value - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if [ -z "${shunit_condition_}" ]; then - # null condition - shunit_return=${SHUNIT_FALSE} - elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # possible return value. treating 0 as true, and non-zero as false. - [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE} - else - # (hopefully) a condition - ( eval ${shunit_condition_} ) >/dev/null 2>&1 - [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # record the test - if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# failure functions -# - -# Records a test failure. -# -# Args: -# message: string: failure message [optional] -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -fail() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 1 ]; then - _shunit_error "fail() requires zero or one arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 1 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_}" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -_FAIL_='eval fail --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${SHUNIT_FALSE} -} -_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' - -# Records a test failure, stating two values should have been the same. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# This is functionally equivalent to calling failNotEquals(). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - failNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# skipping functions -# - -# Force remaining assert and fail functions to be "skipped". -# -# This function forces the remaining assert and fail functions to be "skipped", -# i.e. they will have no effect. Each function skipped will be recorded so that -# the total of asserts and fails will not be altered. -# -# Args: -# None -startSkipping() -{ - __shunit_skip=${SHUNIT_TRUE} -} - -# Resume the normal recording behavior of assert and fail calls. -# -# Args: -# None -endSkipping() -{ - __shunit_skip=${SHUNIT_FALSE} -} - -# Returns the state of assert and fail call skipping. -# -# Args: -# None -# Returns: -# boolean: (TRUE/FALSE constant) -isSkipping() -{ - return ${__shunit_skip} -} - -#----------------------------------------------------------------------------- -# suite functions -# - -# Stub. This function should contains all unit test calls to be made. -# -# DEPRECATED (as of 2.1.0) -# -# This function can be optionally overridden by the user in their test suite. -# -# If this function exists, it will be called when shunit2 is sourced. If it -# does not exist, shunit2 will search the parent script for all functions -# beginning with the word 'test', and they will be added dynamically to the -# test suite. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Adds a function name to the list of tests schedule for execution. -# -# This function should only be called from within the suite() function. -# -# Args: -# function: string: name of a function to add to current unit test suite -suite_addTest() -{ - shunit_func_=${1:-} - - __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" - __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` - - unset shunit_func_ -} - -# Stub. This function will be called once before any tests are run. -# -# Common one-time environment preparation tasks shared by all tests can be -# defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called once after all tests are finished. -# -# Common one-time environment cleanup tasks shared by all tests can be defined -# here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called before each test is run. -# -# Common environment preparation tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#setUp() { :; } - -# Note: see _shunit_mktempFunc() for actual implementation -# Stub. This function will be called after each test is run. -# -# Common environment cleanup tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -#------------------------------------------------------------------------------ -# internal shUnit2 functions -# - -# Create a temporary directory to store various run-time files in. -# -# This function is a cross-platform temporary directory creation tool. Not all -# OSes have the mktemp function, so one is included here. -# -# Args: -# None -# Outputs: -# string: the temporary directory that was created -_shunit_mktempDir() -{ - # try the standard mktemp function - ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return - - # the standard mktemp didn't work. doing our own. - if [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then - _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" -#! /bin/sh -exit ${SHUNIT_TRUE} -EOF - chmod +x "${_shunit_file_}" - done - - unset _shunit_file_ -} - -# Final cleanup function to leave things as we found them. -# -# Besides removing the temporary directory, this function is in charge of the -# final exit code of the unit test. The exit code is based on how the script -# was ended (e.g. normal exit, or via Ctrl-C). -# -# Args: -# name: string: name of the trap called (specified when trap defined) -_shunit_cleanup() -{ - _shunit_name_=$1 - - case ${_shunit_name_} in - EXIT) _shunit_signal_=0 ;; - INT) _shunit_signal_=2 ;; - TERM) _shunit_signal_=15 ;; - *) - _shunit_warn "unrecognized trap value (${_shunit_name_})" - _shunit_signal_=0 - ;; - esac - - # do our work - rm -fr "${__shunit_tmpDir}" - - # exit for all non-EXIT signals - if [ ${_shunit_name_} != 'EXIT' ]; then - _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" - # disable EXIT trap - trap 0 - # add 128 to signal and exit - exit `expr ${_shunit_signal_} + 128` - elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then - _shunit_assertFail 'Unknown failure encountered running a test' - _shunit_generateReport - exit ${SHUNIT_ERROR} - fi - - unset _shunit_name_ _shunit_signal_ -} - -# The actual running of the tests happens here. -# -# Args: -# None -_shunit_execSuite() -{ - for _shunit_test_ in ${__shunit_suite}; do - __shunit_testSuccess=${SHUNIT_TRUE} - - # disable skipping - endSkipping - - # execute the per-test setup function - setUp - - # execute the test - echo "${_shunit_test_}" - eval ${_shunit_test_} - - # execute the per-test tear-down function - tearDown - - # update stats - if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then - __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` - else - __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` - fi - done - - unset _shunit_test_ -} - -# Generates the user friendly report with appropriate OK/FAILED message. -# -# Args: -# None -# Output: -# string: the report of successful and failed tests, as well as totals. -_shunit_generateReport() -{ - _shunit_ok_=${SHUNIT_TRUE} - - # if no exit code was provided one, determine an appropriate one - [ ${__shunit_testsFailed} -gt 0 \ - -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ - && _shunit_ok_=${SHUNIT_FALSE} - - echo - if [ ${__shunit_testsTotal} -eq 1 ]; then - echo "Ran ${__shunit_testsTotal} test." - else - echo "Ran ${__shunit_testsTotal} tests." - fi - - _shunit_failures_='' - _shunit_skipped_='' - [ ${__shunit_assertsFailed} -gt 0 ] \ - && _shunit_failures_="failures=${__shunit_assertsFailed}" - [ ${__shunit_assertsSkipped} -gt 0 ] \ - && _shunit_skipped_="skipped=${__shunit_assertsSkipped}" - - if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then - _shunit_msg_='OK' - [ -n "${_shunit_skipped_}" ] \ - && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})" - else - _shunit_msg_="FAILED (${_shunit_failures_}" - [ -n "${_shunit_skipped_}" ] \ - && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}" - _shunit_msg_="${_shunit_msg_})" - fi - - echo - echo ${_shunit_msg_} - __shunit_reportGenerated=${SHUNIT_TRUE} - - unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_ -} - -# Test for whether a function should be skipped. -# -# Args: -# None -# Returns: -# boolean: whether the test should be skipped (TRUE/FALSE constant) -_shunit_shouldSkip() -{ - [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} - _shunit_assertSkip -} - -# Records a successful test. -# -# Args: -# None -_shunit_assertPass() -{ - __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Records a test failure. -# -# Args: -# message: string: failure message to provide user -_shunit_assertFail() -{ - _shunit_msg_=$1 - - __shunit_testSuccess=${SHUNIT_FALSE} - __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` - echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}" - - unset _shunit_msg_ -} - -# Records a skipped test. -# -# Args: -# None -_shunit_assertSkip() -{ - __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Prepare a script filename for sourcing. -# -# Args: -# script: string: path to a script to source -# Returns: -# string: filename prefixed with ./ (if necessary) -_shunit_prepForSourcing() -{ - _shunit_script_=$1 - case "${_shunit_script_}" in - /*|./*) echo "${_shunit_script_}" ;; - *) echo "./${_shunit_script_}" ;; - esac - unset _shunit_script_ -} - -# Escape a character in a string. -# -# Args: -# c: string: unescaped character -# s: string: to escape character in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharInStr() -{ - [ -n "$2" ] || return # no point in doing work on an empty string - - # Note: using shorter variable names to prevent conflicts with - # _shunit_escapeCharactersInString(). - _shunit_c_=$1 - _shunit_s_=$2 - - - # escape the character - echo ''${_shunit_s_}'' |sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' - - unset _shunit_c_ _shunit_s_ -} - -# Escape a character in a string. -# -# Args: -# str: string: to escape characters in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharactersInString() -{ - [ -n "$1" ] || return # no point in doing work on an empty string - - _shunit_str_=$1 - - # Note: using longer variable names to prevent conflicts with - # _shunit_escapeCharInStr(). - for _shunit_char_ in '"' '$' "'" '`'; do - _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` - done - - echo "${_shunit_str_}" - unset _shunit_char_ _shunit_str_ -} - -# Extract list of functions to run tests against. -# -# Args: -# script: string: name of script to extract functions from -# Returns: -# string: of function names -_shunit_extractTestFunctions() -{ - _shunit_script_=$1 - - # extract the lines with test function names, strip of anything besides the - # function name, and output everything on a single line. - _shunit_regex_='^[ ]*(function )*test[A-Za-z0-9_]* *\(\)' - egrep "${_shunit_regex_}" "${_shunit_script_}" \ - |sed 's/^[^A-Za-z0-9_]*//;s/^function //;s/\([A-Za-z0-9_]*\).*/\1/g' \ - |xargs - - unset _shunit_regex_ _shunit_script_ -} - -#------------------------------------------------------------------------------ -# main -# - -# determine the operating mode -if [ $# -eq 0 ]; then - __shunit_script=${__SHUNIT_PARENT} - __shunit_mode=${__SHUNIT_MODE_SOURCED} -else - __shunit_script=$1 - [ -r "${__shunit_script}" ] || \ - _shunit_fatal "unable to read from ${__shunit_script}" - __shunit_mode=${__SHUNIT_MODE_STANDALONE} -fi - -# create a temporary storage location -__shunit_tmpDir=`_shunit_mktempDir` - -# provide a public temporary directory for unit test scripts -# TODO(kward): document this -SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" -mkdir "${SHUNIT_TMPDIR}" - -# setup traps to clean up after ourselves -trap '_shunit_cleanup EXIT' 0 -trap '_shunit_cleanup INT' 2 -trap '_shunit_cleanup TERM' 15 - -# create phantom functions to work around issues with Cygwin -_shunit_mktempFunc -PATH="${__shunit_tmpDir}:${PATH}" - -# make sure phantom functions are executable. this will bite if /tmp (or the -# current $TMPDIR) points to a path on a partition that was mounted with the -# 'noexec' option. the noexec command was created with _shunit_mktempFunc(). -noexec 2>/dev/null || _shunit_fatal \ - 'please declare TMPDIR with path on partition with exec permission' - -# we must manually source the tests in standalone mode -if [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then - . "`_shunit_prepForSourcing \"${__shunit_script}\"`" -fi - -# execute the oneTimeSetUp function (if it exists) -oneTimeSetUp - -# execute the suite function defined in the parent test script -# deprecated as of 2.1.0 -suite - -# if no suite function was defined, dynamically build a list of functions -if [ -z "${__shunit_suite}" ]; then - shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` - for shunit_func_ in ${shunit_funcs_}; do - suite_addTest ${shunit_func_} - done -fi -unset shunit_func_ shunit_funcs_ - -# execute the tests -_shunit_execSuite - -# execute the oneTimeTearDown function (if it exists) -oneTimeTearDown - -# generate the report -_shunit_generateReport - -# that's it folks -[ ${__shunit_testsFailed} -eq 0 ] -exit $? diff --git a/tests/shunit2/shunit2_test.sh b/tests/shunit2/shunit2_test.sh deleted file mode 100755 index d8f5a9c..0000000 --- a/tests/shunit2/shunit2_test.sh +++ /dev/null @@ -1,124 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit test suite runner. -# -# This script runs all the unit tests that can be found, and generates a nice -# report of the tests. - -MY_NAME=`basename $0` -MY_PATH=`dirname $0` - -PREFIX='shunit2_test_' -SHELLS='/bin/sh /bin/bash /bin/dash /bin/ksh /bin/pdksh /bin/zsh' -TESTS='' -for test in ${PREFIX}[a-z]*.sh; do - TESTS="${TESTS} ${test}" -done - -# load common unit test functions -. ../lib/versions -. ./shunit2_test_helpers - -usage() -{ - echo "usage: ${MY_NAME} [-e key=val ...] [-s shell(s)] [-t test(s)]" -} - -env='' - -# process command line flags -while getopts 'e:hs:t:' opt; do - case ${opt} in - e) # set an environment variable - key=`expr "${OPTARG}" : '\([^=]*\)='` - val=`expr "${OPTARG}" : '[^=]*=\(.*\)'` - if [ -z "${key}" -o -z "${val}" ]; then - usage - exit 1 - fi - eval "${key}='${val}'" - export ${key} - env="${env:+${env} }${key}" - ;; - h) usage; exit 0 ;; # output help - s) shells=${OPTARG} ;; # list of shells to run - t) tests=${OPTARG} ;; # list of tests to run - *) usage; exit 1 ;; - esac -done -shift `expr ${OPTIND} - 1` - -# fill shells and/or tests -shells=${shells:-${SHELLS}} -tests=${tests:-${TESTS}} - -# error checking -if [ -z "${tests}" ]; then - th_error 'no tests found to run; exiting' - exit 1 -fi - -cat <&1; ) - done -done diff --git a/tests/shunit2/shunit2_test_asserts.sh b/tests/shunit2/shunit2_test_asserts.sh deleted file mode 100755 index 38647ec..0000000 --- a/tests/shunit2/shunit2_test_asserts.sh +++ /dev/null @@ -1,206 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit test for assert functions - -# load test helpers -. ./shunit2_test_helpers - -#------------------------------------------------------------------------------ -# suite tests -# - -commonEqualsSame() -{ - fn=$1 - - ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'equal' $? "${stdoutF}" "${stderrF}" - - ( ${fn} "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'equal; with msg' $? "${stdoutF}" "${stderrF}" - - ( ${fn} 'abc def' 'abc def' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'equal with spaces' $? "${stdoutF}" "${stderrF}" - - ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'not equal' $? "${stdoutF}" "${stderrF}" - - ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'null values' $? "${stdoutF}" "${stderrF}" - - ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -commonNotEqualsSame() -{ - fn=$1 - - ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not same' $? "${stdoutF}" "${stderrF}" - - ( ${fn} "${MSG}" 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not same, with msg' $? "${stdoutF}" "${stderrF}" - - ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}" - - ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}" - - ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -testAssertEquals() -{ - commonEqualsSame 'assertEquals' -} - -testAssertNotEquals() -{ - commonNotEqualsSame 'assertNotEquals' -} - -testAssertSame() -{ - commonEqualsSame 'assertSame' -} - -testAssertNotSame() -{ - commonNotEqualsSame 'assertNotSame' -} - -testAssertNull() -{ - ( assertNull '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'null' $? "${stdoutF}" "${stderrF}" - - ( assertNull "${MSG}" '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'null, with msg' $? "${stdoutF}" "${stderrF}" - - ( assertNull 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'not null' $? "${stdoutF}" "${stderrF}" - - ( assertNull >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( assertNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -testAssertNotNull() -{ - ( assertNotNull 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not null' $? "${stdoutF}" "${stderrF}" - - ( assertNotNull "${MSG}" 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not null, with msg' $? "${stdoutF}" "${stderrF}" - - ( assertNotNull 'x"b' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not null, with double-quote' $? \ - "${stdoutF}" "${stderrF}" - - ( assertNotNull "x'b" >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not null, with single-quote' $? \ - "${stdoutF}" "${stderrF}" - - ( assertNotNull 'x$b' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not null, with dollar' $? \ - "${stdoutF}" "${stderrF}" - - ( assertNotNull 'x`b' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'not null, with backtick' $? \ - "${stdoutF}" "${stderrF}" - - ( assertNotNull '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}" - - # there is no test for too few arguments as $1 might actually be null - - ( assertNotNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -testAssertTrue() -{ - ( assertTrue 0 >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'true' $? "${stdoutF}" "${stderrF}" - - ( assertTrue "${MSG}" 0 >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'true, with msg' $? "${stdoutF}" "${stderrF}" - - ( assertTrue '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'true condition' $? "${stdoutF}" "${stderrF}" - - ( assertTrue 1 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'false' $? "${stdoutF}" "${stderrF}" - - ( assertTrue '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'false condition' $? "${stdoutF}" "${stderrF}" - - ( assertTrue '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}" - - ( assertTrue >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( assertTrue arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -testAssertFalse() -{ - ( assertFalse 1 >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'false' $? "${stdoutF}" "${stderrF}" - - ( assertFalse "${MSG}" 1 >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'false, with msg' $? "${stdoutF}" "${stderrF}" - - ( assertFalse '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" ) - th_assertTrueWithNoOutput 'false condition' $? "${stdoutF}" "${stderrF}" - - ( assertFalse 0 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'true' $? "${stdoutF}" "${stderrF}" - - ( assertFalse '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}" - - ( assertFalse '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}" - - ( assertFalse >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( assertFalse arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -#------------------------------------------------------------------------------ -# suite functions -# - -oneTimeSetUp() -{ - th_oneTimeSetUp - - MSG='This is a test message' -} - -# load and run shUnit2 -[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 -. ${TH_SHUNIT} diff --git a/tests/shunit2/shunit2_test_failures.sh b/tests/shunit2/shunit2_test_failures.sh deleted file mode 100755 index 933a0b1..0000000 --- a/tests/shunit2/shunit2_test_failures.sh +++ /dev/null @@ -1,86 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit test for failure functions - -# load common unit-test functions -. ./shunit2_test_helpers - -#----------------------------------------------------------------------------- -# suite tests -# - -testFail() -{ - ( fail >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'fail' $? "${stdoutF}" "${stderrF}" - - ( fail "${MSG}" >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'fail with msg' $? "${stdoutF}" "${stderrF}" - - ( fail arg1 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -testFailNotEquals() -{ - ( failNotEquals 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}" - - ( failNotEquals "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}" - - ( failNotEquals 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}" - - ( failNotEquals '' '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}" - - ( failNotEquals >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( failNotEquals arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -testFailSame() -{ - ( failSame 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}" - - ( failSame "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}" - - ( failSame 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}" - - ( failSame '' '' >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}" - - ( failSame >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}" - - ( failSame arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" ) - th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}" -} - -#----------------------------------------------------------------------------- -# suite functions -# - -oneTimeSetUp() -{ - th_oneTimeSetUp - - MSG='This is a test message' -} - -# load and run shUnit2 -[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 -. ${TH_SHUNIT} diff --git a/tests/shunit2/shunit2_test_helpers b/tests/shunit2/shunit2_test_helpers deleted file mode 100644 index a9989d6..0000000 --- a/tests/shunit2/shunit2_test_helpers +++ /dev/null @@ -1,229 +0,0 @@ -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit test common functions - -# treat unset variables as an error when performing parameter expansion -set -u - -# set shwordsplit for zsh -[ -n "${ZSH_VERSION:-}" ] && setopt shwordsplit - -# -# constants -# - -# path to shUnit2 library. can be overridden by setting SHUNIT_INC -TH_SHUNIT=${SHUNIT_INC:-./shunit2} - -# configure debugging. set the DEBUG environment variable to any -# non-empty value to enable debug output, or TRACE to enable trace -# output. -TRACE=${TRACE:+'th_trace '} -[ -n "${TRACE}" ] && DEBUG=1 -[ -z "${TRACE}" ] && TRACE=':' - -DEBUG=${DEBUG:+'th_debug '} -[ -z "${DEBUG}" ] && DEBUG=':' - -# -# variables -# - -th_RANDOM=0 - -# -# functions -# - -# message functions -th_trace() { echo "${MY_NAME}:TRACE $@" >&2; } -th_debug() { echo "${MY_NAME}:DEBUG $@" >&2; } -th_info() { echo "${MY_NAME}:INFO $@" >&2; } -th_warn() { echo "${MY_NAME}:WARN $@" >&2; } -th_error() { echo "${MY_NAME}:ERROR $@" >&2; } -th_fatal() { echo "${MY_NAME}:FATAL $@" >&2; } - -# output subtest name -th_subtest() { echo " $@" >&2; } - -th_oneTimeSetUp() -{ - # these files will be cleaned up automatically by shUnit2 - stdoutF="${SHUNIT_TMPDIR}/stdout" - stderrF="${SHUNIT_TMPDIR}/stderr" - returnF="${SHUNIT_TMPDIR}/return" - expectedF="${SHUNIT_TMPDIR}/expected" -} - -# generate a random number -th_generateRandom() -{ - tfgr_random=${th_RANDOM} - - while [ "${tfgr_random}" = "${th_RANDOM}" ]; do - if [ -n "${RANDOM:-}" ]; then - # $RANDOM works - tfgr_random=${RANDOM}${RANDOM}${RANDOM}$$ - elif [ -r '/dev/urandom' ]; then - tfgr_random=`od -vAn -N4 -tu4 >> STDOUT' >&2 - cat "${_th_stdout_}" >&2 - fi - if [ -n "${_th_stderr_}" -a -s "${_th_stderr_}" ]; then - echo '>>> STDERR' >&2 - cat "${_th_stderr_}" >&2 - fi - if [ -n "${_th_stdout_}" -o -n "${_th_stderr_}" ]; then - echo '<<< end output' >&2 - fi - fi - - unset _th_return_ _th_stdout_ _th_stderr_ -} - -# -# main -# - -${TRACE} 'trace output enabled' -${DEBUG} 'debug output enabled' diff --git a/tests/shunit2/shunit2_test_macros.sh b/tests/shunit2/shunit2_test_macros.sh deleted file mode 100755 index ce57b14..0000000 --- a/tests/shunit2/shunit2_test_macros.sh +++ /dev/null @@ -1,246 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit test for macros. - -# load test helpers -. ./shunit2_test_helpers - -#------------------------------------------------------------------------------ -# suite tests -# - -testAssertEquals() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_EQUALS_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_EQUALS_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testAssertNotEquals() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_NOT_EQUALS_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NOT_EQUALS_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_NOT_EQUALS_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NOT_EQUALS_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testSame() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_SAME_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_SAME_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testNotSame() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_NOT_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NOT_SAME_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_NOT_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NOT_SAME_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testNull() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_NULL_} 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NULL_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_NULL_} '"some msg"' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NULL_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testNotNull() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_NOT_NULL_} '' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NOT_NULL_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_NOT_NULL_} '"some msg"' '""' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_NOT_NULL_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stdoutF}" "${stderrF}" >&2 -} - -testAssertTrue() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_TRUE_} ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_TRUE_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - - ( ${_ASSERT_TRUE_} '"some msg"' ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_TRUE_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testAssertFalse() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_ASSERT_FALSE_} ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_FALSE_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_ASSERT_FALSE_} '"some msg"' ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_ASSERT_FALSE_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testFail() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_FAIL_} >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_FAIL_} '"some msg"' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testFailNotEquals() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_FAIL_NOT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_NOT_EQUALS_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_FAIL_NOT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_NOT_EQUALS_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testFailSame() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_FAIL_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_SAME_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_FAIL_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_SAME_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testFailNotSame() -{ - # start skipping if LINENO not available - [ -z "${LINENO:-}" ] && startSkipping - - ( ${_FAIL_NOT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_NOT_SAME_ failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 - - ( ${_FAIL_NOT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" ) - grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null - rtrn=$? - assertTrue '_FAIL_NOT_SAME_ w/ msg failure' ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -#------------------------------------------------------------------------------ -# suite functions -# - -oneTimeSetUp() -{ - th_oneTimeSetUp -} - -# load and run shUnit2 -[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 -. ${TH_SHUNIT} diff --git a/tests/shunit2/shunit2_test_misc.sh b/tests/shunit2/shunit2_test_misc.sh deleted file mode 100755 index d264628..0000000 --- a/tests/shunit2/shunit2_test_misc.sh +++ /dev/null @@ -1,160 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit tests of miscellaneous things - -# load test helpers -. ./shunit2_test_helpers - -#------------------------------------------------------------------------------ -# suite tests -# - -# Note: the test script is prefixed with '#' chars so that shUnit2 does not -# incorrectly interpret the embedded functions as real functions. -testUnboundVariable() -{ - unittestF="${SHUNIT_TMPDIR}/unittest" - sed 's/^#//' >"${unittestF}" <"${stdoutF}" 2>"${stderrF}" ) - assertFalse 'expected a non-zero exit value' $? - grep '^ASSERT:Unknown failure' "${stdoutF}" >/dev/null - assertTrue 'assert message was not generated' $? - grep '^Ran [0-9]* test' "${stdoutF}" >/dev/null - assertTrue 'test count message was not generated' $? - grep '^FAILED' "${stdoutF}" >/dev/null - assertTrue 'failure message was not generated' $? -} - -testIssue7() -{ - ( assertEquals 'Some message.' 1 2 >"${stdoutF}" 2>"${stderrF}" ) - diff "${stdoutF}" - >/dev/null < but was:<2> -EOF - rtrn=$? - assertEquals ${SHUNIT_TRUE} ${rtrn} - [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2 -} - -testPrepForSourcing() -{ - assertEquals '/abc' `_shunit_prepForSourcing '/abc'` - assertEquals './abc' `_shunit_prepForSourcing './abc'` - assertEquals './abc' `_shunit_prepForSourcing 'abc'` -} - -testEscapeCharInStr() -{ - actual=`_shunit_escapeCharInStr '\' ''` - assertEquals '' "${actual}" - assertEquals 'abc\\' `_shunit_escapeCharInStr '\' 'abc\'` - assertEquals 'abc\\def' `_shunit_escapeCharInStr '\' 'abc\def'` - assertEquals '\\def' `_shunit_escapeCharInStr '\' '\def'` - - actual=`_shunit_escapeCharInStr '"' ''` - assertEquals '' "${actual}" - assertEquals 'abc\"' `_shunit_escapeCharInStr '"' 'abc"'` - assertEquals 'abc\"def' `_shunit_escapeCharInStr '"' 'abc"def'` - assertEquals '\"def' `_shunit_escapeCharInStr '"' '"def'` - - actual=`_shunit_escapeCharInStr '$' ''` - assertEquals '' "${actual}" - assertEquals 'abc\$' `_shunit_escapeCharInStr '$' 'abc$'` - assertEquals 'abc\$def' `_shunit_escapeCharInStr '$' 'abc$def'` - assertEquals '\$def' `_shunit_escapeCharInStr '$' '$def'` - -# actual=`_shunit_escapeCharInStr "'" ''` -# assertEquals '' "${actual}" -# assertEquals "abc\\'" `_shunit_escapeCharInStr "'" "abc'"` -# assertEquals "abc\\'def" `_shunit_escapeCharInStr "'" "abc'def"` -# assertEquals "\\'def" `_shunit_escapeCharInStr "'" "'def"` - -# # must put the backtick in a variable so the shell doesn't misinterpret it -# # while inside a backticked sequence (e.g. `echo '`'` would fail). -# backtick='`' -# actual=`_shunit_escapeCharInStr ${backtick} ''` -# assertEquals '' "${actual}" -# assertEquals '\`abc' \ -# `_shunit_escapeCharInStr "${backtick}" ${backtick}'abc'` -# assertEquals 'abc\`' \ -# `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}` -# assertEquals 'abc\`def' \ -# `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}'def'` -} - -testEscapeCharInStr_specialChars() -{ - # make sure our forward slash doesn't upset sed - assertEquals '/' `_shunit_escapeCharInStr '\' '/'` - - # some shells escape these differently - #assertEquals '\\a' `_shunit_escapeCharInStr '\' '\a'` - #assertEquals '\\b' `_shunit_escapeCharInStr '\' '\b'` -} - -# Test the various ways of declaring functions. -# -# Prefixing (then stripping) with comment symbol so these functions aren't -# treated as real functions by shUnit2. -testExtractTestFunctions() -{ - f="${SHUNIT_TMPDIR}/extract_test_functions" - sed 's/^#//' <"${f}" -#testABC() { echo 'ABC'; } -#test_def() { -# echo 'def' -#} -#testG3 () -#{ -# echo 'G3' -#} -#function test4() { echo '4'; } -# test5() { echo '5'; } -#some_test_function() { echo 'some func'; } -#func_with_test_vars() { -# testVariable=1234 -#} -EOF - - actual=`_shunit_extractTestFunctions "${f}"` - assertEquals 'testABC test_def testG3 test4 test5' "${actual}" -} - -#------------------------------------------------------------------------------ -# suite functions -# - -setUp() -{ - for f in ${expectedF} ${stdoutF} ${stderrF}; do - cp /dev/null ${f} - done -} - -oneTimeSetUp() -{ - th_oneTimeSetUp -} - -# load and run shUnit2 -[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0 -. ${TH_SHUNIT} diff --git a/tests/shunit2/shunit2_test_standalone.sh b/tests/shunit2/shunit2_test_standalone.sh deleted file mode 100755 index 2ac4725..0000000 --- a/tests/shunit2/shunit2_test_standalone.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/sh -# $Id$ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2010 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 unit test for standalone operation. -# -# This unit test is purely to test that calling shunit2 directly, while passing -# the name of a unit test script, works. When run, this script determines if it -# is running as a standalone program, and calls main() if it is. - -ARGV0=`basename "$0"` - -# load test helpers -. ./shunit2_test_helpers - -#------------------------------------------------------------------------------ -# suite tests -# - -testStandalone() -{ - assertTrue ${SHUNIT_TRUE} -} - -#------------------------------------------------------------------------------ -# main -# - -main() -{ - ${TH_SHUNIT} "${ARGV0}" -} - -# are we running as a standalone? -if [ "${ARGV0}" = 'shunit2_test_standalone.sh' ]; then - if [ $# -gt 0 ]; then main "$@"; else main; fi -fi From 3ae3005acede7cc0b179e747280a411247188e26 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 10:27:36 +0200 Subject: [PATCH 11/16] Updated sync file --- sync.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sync.conf b/sync.conf index 87203f7..e05e780 100644 --- a/sync.conf +++ b/sync.conf @@ -2,7 +2,7 @@ ###### osync - Rsync based two way sync engine with fault tolerance ###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr) -###### osync v1.1x / v1.2x config file rev 2016080902 +###### osync v1.1x / v1.2x config file rev 2016081901 ## ---------- GENERAL OPTIONS @@ -80,7 +80,7 @@ REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com" PRESERVE_PERMISSIONS=yes PRESERVE_OWNER=yes PRESERVE_GROUP=yes -## Does not work and will be ignored on MacOS X +## On MACOS X, does not work and will be ignored PRESERVE_EXECUTABILITY=yes ## Preserve ACLS. Make sure source and target FS can manage same ACLs or you'll get loads of errors. @@ -104,6 +104,9 @@ RSYNC_COMPRESS=yes SOFT_MAX_EXEC_TIME=7200 HARD_MAX_EXEC_TIME=10600 +## Log a message every KEEP_LOGGING seconds just to know the task is still alive +KEEP_LOGGING=1801 + ## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish. MIN_WAIT=60 From d79f53760b968a16604f4858e8d34f07b667615b Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Aug 2016 10:31:47 +0200 Subject: [PATCH 12/16] Updated coding style file --- CODING_STYLE.TXT | 95 +++++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/CODING_STYLE.TXT b/CODING_STYLE.TXT index 524c639..da88698 100644 --- a/CODING_STYLE.TXT +++ b/CODING_STYLE.TXT @@ -1,4 +1,4 @@ -Coding style used for my bash projects (v2.1 Oct 2015) +Coding style used for my bash projects (v2.2 Aug 2016) ++++++ Header @@ -8,7 +8,7 @@ Always use the following header #!/usr/bin/env bash PROGRAM="program-name" # Long description -AUTHOR="(C) 20XX-20YY by Orsiris \"Ozy\" de Jong" +AUTHOR="(C) 20XX-20YY by Orsiris de Jong" CONTACT="http://www.example.com me@example.com" PROGRAM_BUILD=YYYYMMDDVV @@ -23,8 +23,8 @@ YYYYMMDDVV (Year, Month, Day, Revision): Example: 2015012402 = 2nd revision of 2 Change old scripts with for i in $(grep -r '#!/bin/bash' * |cut -f1 -d':'); do sed -i 's&#!/bin/bash&#!/usr/bin/env bash&g' $i; done - type instead of type -p for bash test (other shells don't know -p) + ++++++ Indentation Using tabs @@ -38,59 +38,78 @@ Some command # comment ++++++ Work comments -Whenever there is some idea to postpone, use #TODO[-version]:[dev-name:] some remark -A marker must be left where on the line a dev is working (when the work isn't finished). Marker is #WIP:dev-name: some remark +Whenever there is some idea to postpone, use #TODO(priority):[dev-name:] some remark +Priority can be critical, high, medium, low, verylow. No release can happen if there are TODOs other than low or verylow. +Example: #TODO(high):deajan: need to do something + +A "work in progress" marker must be left on the line a dev is working when it's work isn't finished). Marker is #WIP:dev-name: some remark dev-name is mandatory if more than one person is coding -Example: #TODO-v2.1:deajan: need to do something +Example: #WIP:deajan: missing function something ++++++ Variables -All local variables are lowercase, separated by _ (ex: low_wait) -All global variables full upercase, separated by _ (ex: EXEC_TIME) -All environment variables (verbose, silent, debug, etc) have prefix _ and are full upercase, separated by _ (ex: _PARANOIA_DEBUG) +All local variables names have each first letter of the words uppercase and all others lowercase, except for the first word where all letters are lowercase +Example: someLongVariable +All global variables are full upercase, separated by _ +Example: EXEC_TIME +All environment variables (verbose, silent, debug, etc) have prefix _ and are full upercase, separated by _ +Example: _PARANOIA_DEBUG ++++++ Functions -Every word in a function begins with an uppercase (ex: SomeFunctionDoesThings) +All function names should begin with an uppercase letter for every word, the other letters should be lowercase +Example: SomeFunctionThatRocks -Define functions this way. Use sed ':a;N;$!ba;s/\n{\n/ {\n/g' to adapt when opening bracket is on a new line. +Bash does not provide any checks against missing function arguments. Also, missing quotes can lead to an inconsistent number of arguments. +Most functions should have a first line that calls the special function __CheckArguments, which checks the number of given arguments for a function in order +to find possible problems. Number of arguments are given as first argument to __CheckArguments. +__CheckArguments will only trigger when the script is launched with _PARANOIA_DEBUG=yes. Also, it will only exist in the debug version. +Use the following convention for function definition: -function something { - +function SomeFunction { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + ... } -If function has some arguments, use local variable names that are more readable than $1...$n. Explain via comments what those variables contain if needed. +Use sed ':a;N;$!ba;s/\n{\n/ {\n/g' to convert functions that have opening brackets on a new line. -function anotherthing { - local var_name="${1}" - local other_var_name="${2}" # This variable contains stuff +If the function has arguments, use local variable names that are more readable than $1...$n. Explain via comments what those variables contain if needed. +Declare arguments before launching __CheckArguments: + +function AnotherFunction { + local varName="${1}" + local otherVarName="${2}" # This variable contains stuff + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + ... } Functions should always have return status -function thirdthing { - some_command +function RandomFunction { + ... return $? } ++++++ Sub functions -When a function is a subroutine of another function, it is called _SomethingAsSubFunction +When a function is a subroutine of another function, it is called _SomethingAsSubFunction: +Example: -++++++ Function argument check +function _ApplyLocally +function _ApplyRemotely +function Apply -Bash does not provide any checks against missing function arguments. Also, missing quotes can lead to an inconsistent number of arguments. -Every function call will be checked by __CheckArguments which takes the number of arguments, $# (the real number of args given), the parent function name and the parent function's arguments. -__CheckArguments will trigger a critical error if number of arguments if incorrect. This will also prevent silent typo errors. -Ex: +++++++ For and While statements -function Something { - local some="${1}" - local other="${2}" - local args="${3}" - __CheckArguments 3 $# $FUNCNAME "$*" +For and while statements will have the "do" part on the first line. +Example: -__CheckArguments will only trigger if script is called with DEBUG=yes -Also, with PARANOIA_DEBUG=yes, __CheckArguments will recount all arguments given by "$*" and compare. This can mislead if arguments contain spaces. +for i in "${var[@]}"; do + ... +done + +while [ $i -eq 1 ]; do + ... +done ++++++ If statements @@ -106,7 +125,7 @@ fi A logging function is available with the following levels of logging: -- DEBUG: Only log this when DEBUG flas is set in program. Any command forged for eval should be logged by this. +- DEBUG: Only log this when DEBUG flas is set in program. Any command forged for eval instruction should be logged by this. - NOTICE: Standard messages - WARN: Requires attention - ERROR: Program produced an error but continues execution @@ -128,7 +147,7 @@ cmd=$SSH_CMD' "some; commands \"'$VARIABLE'\" some; other; commands" > some_file ++++++ File variables All eval cmd should exit their content to a file called "$RUNDIR/osync.$FUNCNAME.$SCRIPT_PID" -Dots are used instead of '_' so variables can be separated with a forbidden char in variables, so they get detected. +Dots are used instead of '_' so variables can be separated with a forbidden char in variable names, so the separtors apply as wished. ++++++ Finding code errors @@ -154,3 +173,11 @@ function FunctionName { #__ENDFUNC These functions are inserted into code that has placeholders like #__FUNC:FuncName + ++++++++ Exit codes + +Normal exit code = 0 +Run with errors exit code = 1 +Run with warnings exit code = 2 +Wrong shell exit code = 127 +Usage function exit code = 128 From adf742775d83216b6e787f760b5e144ffb3059c9 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 11:30:28 +0200 Subject: [PATCH 13/16] Updated changelog & coding style --- CHANGELOG.md | 3 +++ CODING_STYLE.TXT | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8473c..ec46a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ KNOWN ISSUES RECENT CHANGES -------------- +- Logs sent by mail are easier to read + - Better subject (currently running or finished run) + - Fixed bogus double log sent in alert mails - Made unix signals posix compliant - Config file upgrade script now updates header ! test if waitfortaskcompletion kill self works okay with osync diff --git a/CODING_STYLE.TXT b/CODING_STYLE.TXT index da88698..12e4c89 100644 --- a/CODING_STYLE.TXT +++ b/CODING_STYLE.TXT @@ -1,4 +1,4 @@ -Coding style used for my bash projects (v2.2 Aug 2016) +Coding style used for my bash projects (v2.3 Sep 2016) ++++++ Header @@ -62,7 +62,7 @@ Example: SomeFunctionThatRocks Bash does not provide any checks against missing function arguments. Also, missing quotes can lead to an inconsistent number of arguments. Most functions should have a first line that calls the special function __CheckArguments, which checks the number of given arguments for a function in order -to find possible problems. Number of arguments are given as first argument to __CheckArguments. +to find possible problems. Number of arguments are given as first argument to __CheckArguments. May be a number or a range, eg 0-2 if the function takes optional arguments. __CheckArguments will only trigger when the script is launched with _PARANOIA_DEBUG=yes. Also, it will only exist in the debug version. Use the following convention for function definition: From 32fbb242596c04edac1972b8db761623e4c8dd72 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 11:32:25 +0200 Subject: [PATCH 14/16] Updated ofunctions with ParallelExec --- dev/ofunctions.sh | 159 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 32 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 1c467b0..32ae6ee 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,9 +1,13 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016082204 -## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr +## FUNC_BUILD=2016082801 +## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## To use in a program, define the following variables: +## PROGRAM=program-name +## INSTANCE_ID=program-instance-name +## _DEBUG=yes/no -## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 @@ -28,6 +32,9 @@ fi ERROR_ALERT=0 WARN_ALERT=0 +# Current log +CURRENT_LOG= + ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -49,6 +56,10 @@ SCRIPT_PID=$$ LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) +if [ "$PROGRAM" == "" ]; then + PROGRAM="ofunctions" +fi + ## Default log file until config file is loaded if [ -w /var/log ]; then LOG_FILE="/var/log/$PROGRAM.log" @@ -89,6 +100,7 @@ function _Logger { local evalue="${3}" # What to log to stderr echo -e "$lvalue" >> "$LOG_FILE" + CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" if [ "$_LOGGER_STDERR" -eq 1 ]; then cat <<< "$evalue" 1>&2 @@ -220,11 +232,14 @@ function KillAllChilds { # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending function SendAlert { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run + + __CheckArguments 0-1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local mail_no_attachment= local attachment_command= local subject= + local body= # Windows specific settings local encryption_string= @@ -253,7 +268,8 @@ function SendAlert { else mail_no_attachment=0 fi - MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE") + body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG" + if [ $ERROR_ALERT -eq 1 ]; then subject="Error alert for $INSTANCE_ID" elif [ $WARN_ALERT -eq 1 ]; then @@ -262,11 +278,17 @@ function SendAlert { subject="Alert for $INSTANCE_ID" fi + if [ $runAlert == true ]; then + subject="Currently runing - $subject" + else + subject="Fnished run - $subject" + fi + if [ "$mail_no_attachment" -eq 0 ]; then attachment_command="-a $ALERT_LOG_FILE" fi if type mutt > /dev/null 2>&1 ; then - echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command + echo "$body" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command if [ $? != 0 ]; then Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN" else @@ -283,10 +305,10 @@ function SendAlert { else attachment_command="" fi - echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS + echo "$body" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS if [ $? != 0 ]; then Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN" - echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS + echo "$body" | $(type -p mail) -s "$subject" $DESTINATION_MAILS if [ $? != 0 ]; then Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN" else @@ -300,7 +322,7 @@ function SendAlert { fi if type sendmail > /dev/null 2>&1 ; then - echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS + echo -e "Subject:$subject\r\n$body" | $(type -p sendmail) $DESTINATION_MAILS if [ $? != 0 ]; then Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN" else @@ -323,7 +345,7 @@ function SendAlert { if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\"" fi - $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string + $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$body" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string if [ $? != 0 ]; then Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" else @@ -339,7 +361,7 @@ function SendAlert { else SMTP_OPTIONS="" fi - $(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 + $(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$body" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 if [ $? != 0 ]; then Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN" else @@ -350,7 +372,7 @@ function SendAlert { # pfSense specific if [ -f /usr/local/bin/mail.php ]; then - echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject" + echo "$body" | /usr/local/bin/mail.php -s="$subject" if [ $? != 0 ]; then Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" else @@ -569,6 +591,8 @@ function joinString { # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Warning: Don't imbricate this function into another run if you plan to use the global variable output + +#TODO check missing local values used here function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. @@ -620,7 +644,7 @@ function WaitForTaskCompletion { if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" soft_alert=1 - SendAlert + SendAlert true fi if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then @@ -633,7 +657,7 @@ function WaitForTaskCompletion { Logger "Could not stop task with pid [$pid]." "ERROR" fi done - SendAlert + SendAlert true errrorcount=$((errorcount+1)) fi fi @@ -652,17 +676,18 @@ function WaitForTaskCompletion { retval=$? if [ $retval -ne 0 ]; then errorcount=$((errorcount+1)) - Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG" if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then - WAIT_FOR_TASK_COMPLETION="$pid:$result" + WAIT_FOR_TASK_COMPLETION="$pid:$retval" else - WAIT_FOR_TASK_COMPLETION=";$pid:$result" + WAIT_FOR_TASK_COMPLETION=";$pid:$retval" fi fi fi done pidsArray=("${newPidsArray[@]}") + # Trivial wait time for bash to not eat up all CPU sleep $SLEEP_TIME done @@ -676,6 +701,66 @@ function WaitForTaskCompletion { fi } +# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs +# Returns the number of non zero exit codes from commands +function ParallelExec { + local numberOfProcesses="${1}" # Number of simultaneous commands to run + local commandsArg="${2}" # Semi-colon separated list of commands + + local pid + local runningPids=0 + local counter=0 + local commandsArray + local pidsArray + local newPidsArray + local retval + local retvalAll=0 + local pidState + local commandsArrayPid + + IFS=';' read -r -a commandsArray <<< "$commandsArg" + + Logger "Runnning ${#commandsArray[@]} commands in $numberOfProcesses simultaneous processes." "DEBUG" + + while [ $counter -lt "${#commandsArray[@]}" ] || [ ${#pidsArray[@]} -gt 0 ]; do + + while [ $counter -lt "${#commandsArray[@]}" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do + Logger "Running command [${commandsArray[$counter]}]." "DEBUG" + eval "${commandsArray[$counter]}" & + pid=$! + pidsArray+=($pid) + commandsArrayPid[$pid]="${commandsArray[$counter]}" + counter=$((counter+1)) + done + + + newPidsArray=() + for pid in "${pidsArray[@]}"; do + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + if kill -0 $pid > /dev/null 2>&1; then + pidState=$(ps -p$pid -o state= 2 > /dev/null) + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + newPidsArray+=($pid) + fi + else + # pid is dead, get it's exit code from wait command + wait $pid + retval=$? + if [ $retval -ne 0 ]; then + Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR" + retvalAll=$((retvalAll+1)) + fi + fi + done + pidsArray=("${newPidsArray[@]}") + + # Trivial wait time for bash to not eat up all CPU + sleep $SLEEP_TIME + done + + return $retvalAll +} + function CleanUp { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -969,7 +1054,7 @@ function CheckConnectivityRemoteHost { if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & - WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot ping $REMOTE_HOST" "ERROR" return 1 @@ -1014,12 +1099,15 @@ function __CheckArguments { # Checks the number of arguments of a function and raises an error if some are missing if [ "$_DEBUG" == "yes" ]; then - local number_of_arguments="${1}" # Number of arguments the tested function should have - local number_of_given_arguments="${2}" # Number of arguments that have been passed - local function_name="${3}" # Function name that called __CheckArguments + local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments + local numberOfGivenArguments="${2}" # Number of arguments that have been passed + local functionName="${3}" # Function name that called __CheckArguments + + local minArgs + local maxArgs if [ "$_PARANOIA_DEBUG" == "yes" ]; then - Logger "Entering function [$function_name]." "DEBUG" + Logger "Entering function [$functionName]." "DEBUG" fi # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) @@ -1027,23 +1115,31 @@ function __CheckArguments { # In order to avoid this, we need to iterate over ${4} and count local iterate=4 - local fetch_arguments=1 - local arg_list="" - while [ $fetch_arguments -eq 1 ]; do + local fetchArguments=1 + local argList="" + local countedArguments + while [ $fetchArguments -eq 1 ]; do cmd='argument=${'$iterate'}' eval $cmd if [ "$argument" = "" ]; then - fetch_arguments=0 + fetchArguments=0 else - arg_list="$arg_list [Argument $(($iterate-3)): $argument]" + argList="$arg_list [Argument $(($iterate-3)): $argument]" iterate=$(($iterate+1)) fi done - local counted_arguments=$((iterate-4)) + countedArguments=$((iterate-4)) - if [ $counted_arguments -ne $number_of_arguments ]; then - Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR" - Logger "Arguments passed: $arg_list" "ERROR" + if [ $(IsNumeric "$numberOfArguments") -eq 1 ]; then + minArgs=$numberOfArguments + maxArgs=$numberOfArguments + else + IFS='-' read minArgs maxArgs <<< "$numberOfArguments" + fi + + if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then + Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" + Logger "Arguments passed: $argList" "ERROR" fi fi } @@ -1287,7 +1383,6 @@ function InitLocalOSSettings { function InitRemoteOSSettings { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - #TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) if [ "$PRESERVE_EXECUTABILITY" != "no" ];then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then From c984a3cb745bfad4fc2db850ad08e2d4a80cfe95 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 12:00:30 +0200 Subject: [PATCH 15/16] Renamed sync.conf to sync.conf.example --- sync.conf => sync.conf.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sync.conf => sync.conf.example (100%) diff --git a/sync.conf b/sync.conf.example similarity index 100% rename from sync.conf rename to sync.conf.example From e9cf493e584c54de4f876ce6a6b7de96bcc44552 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 12:00:43 +0200 Subject: [PATCH 16/16] Updated changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec46a90..e1b7fc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ RECENT CHANGES - Added KillAllChilds function to accept multiple pids - Improved logging -XX xxx 2016: osync v1.1.2 released +28 Aug 2016: osync v1.1.2 released +- Renamed sync.conf to sync.conf.example (thanks to https://github.com/hortimech) +- Fixed RunAfterHook may be executed twice - Fixed soft deletion when SUDO_EXEC is enabled 06 Aug 2016: osync v1.1.1 released