From 9c9f5f01a213011275b92302b42860f3bd73e526 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 14 Nov 2016 19:17:50 +0100 Subject: [PATCH] Rebuilt targets --- dev/debug_osync.sh | 181 +++++++++++++++++++++++++++++---------------- osync.sh | 179 ++++++++++++++++++++++++++++---------------- 2 files changed, 232 insertions(+), 128 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index c54e5be..a717a1f 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-beta2 -PROGRAM_BUILD=2016111401 +PROGRAM_BUILD=2016111403 IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG @@ -45,15 +45,16 @@ IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016111401 +## FUNC_BUILD=2016111402 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name ## INSTANCE_ID=program-instance-name ## _DEBUG=yes/no -## _LOGGER_STDERR=True/False -## _LOGGER_ERR_ONLY=True/False +## _LOGGER_STDERR=true/false +## _LOGGER_ERR_ONLY=true/false +## _LOGGER_PREFIX="date"/"time"/"" #TODO: Windows checks, check sendmail & mailsend @@ -189,7 +190,7 @@ function Logger { WARN_ALERT=true return elif [ "$level" == "NOTICE" ]; then - if [ "$_LOGGER_ERR_ONLY" != True ]; then + if [ "$_LOGGER_ERR_ONLY" != true ]; then _Logger "$prefix$value" fi return @@ -198,6 +199,11 @@ function Logger { _Logger "$prefix$value" fi return + elif [ "$level" == "ALWAYS" ]; then + if [ $_SILENT != true ]; then + _Logger "$prefix$value" "$prefix$level:$value" "$level:$value" + fi + return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then _Logger "$prefix$value" @@ -321,9 +327,9 @@ function SendAlert { eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" if [ $? != 0 ]; then Logger "Cannot create [$ALERT_LOG_FILE]" "WARN" - attachment=False + attachment=false else - attachment=True + attachment=true fi body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG" @@ -341,14 +347,14 @@ function SendAlert { subject="Fnished run - $subject" fi - if [ "$attachment" == True ]; then + if [ "$attachment" == true ]; then attachmentFile="$ALERT_LOG_FILE" fi SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$ENCRYPTION" "SMTP_USER" "$SMTP_PASSWORD" # Delete tmp log file - if [ "$attachment" == True ]; then + if [ "$attachment" == true ]; then if [ -f "$ALERT_LOG_FILE" ]; then rm -f "$ALERT_LOG_FILE" fi @@ -540,7 +546,7 @@ function LoadConfigFile { } function Spinner { - if [ $_SILENT == true ]; then + if [ $_SILENT == true ] || [ $_LOGGER_ERR_ONLY == true ]; then return 0 fi @@ -2562,6 +2568,7 @@ function _deleteLocal { Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir]." "VERBOSE" mv -f "$replicaDir$files" "$replicaDir$deletionDir" fi + echo "$replicaDir$files" >> "$RUN_DIR/$PROGRAM.delete.$replicaType.$SCRIPT_PID" if [ $? != 0 ]; then Logger "Cannot move [$replicaDir$files] to deletion directory." "ERROR" echo "$files" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" @@ -3239,6 +3246,43 @@ function SoftDelete { fi } +function _SummaryFromFile { + local summaryFile="${1}" + local direction="${2}" + + __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + + if [ -f "$summaryFile" ]; then + while read -r file; do + Logger "$direction $(echo $file | awk '{for (i=2; i> TARGET" "ALWAYS" + + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>" + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<" + + Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS" + _SummaryFromFile "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID" ">>" + _SummaryFromFile "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID" "<<" + + Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS" + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>" + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<" + + _LOGGER_PREFIX="$prefix" +} + function Init { __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG @@ -3374,7 +3418,7 @@ function Init { ## Set sync only function arguments for rsync SYNC_OPTS="-u" - if [ $_VERBOSE == true ]; then + if [ $_VERBOSE == true ] || [ $_SUMMARY == true ]; then SYNC_OPTS=$SYNC_OPTS" -i" fi @@ -3435,8 +3479,10 @@ function Usage { echo "" echo "[OPTIONS]" echo "--dry Will run osync without actually doing anything; just testing" + echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--verbose Increases output" echo "--stats Adds rsync transfer statistics to verbose output" echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" @@ -3510,7 +3556,6 @@ fi INITIATOR_LOCK_FILE_EXISTS=false TARGET_LOCK_FILE_EXISTS=false - FORCE_UNLOCK=false no_maxtime=false opts="" @@ -3523,6 +3568,7 @@ _QUICK_SYNC=0 sync_on_changes=false _NOLOCKS=false osync_cmd=$0 +_SUMMARY=false if [ $# -eq 0 ] then @@ -3601,9 +3647,16 @@ for i in "$@"; do --no-locks) _NOLOCKS=true ;; - --only-errors) - _LOGGER_STDERR=True - _LOGGER_ERR_ONLY=True + --errors-only) + #TODO: let err_only only output to stderr + #_LOGGER_STDERR=true + _LOGGER_ERR_ONLY=true + ;; + --summary) + _SUMMARY=true + ;; + --no-prefix) + _LOGGER_PREFIX="" ;; *) if [ $first == "0" ]; then @@ -3657,59 +3710,59 @@ opts="${opts# *}" LoadConfigFile "$ConfigFile" fi - if [ "$LOGFILE" == "" ]; then - if [ -w /var/log ]; then - LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log" - elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then - LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log" - else - LOG_FILE="./$PROGRAM.$INSTANCE_ID.log" - fi +if [ "$LOGFILE" == "" ]; then + if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log" + elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then + LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log" else - LOG_FILE="$LOGFILE" + LOG_FILE="./$PROGRAM.$INSTANCE_ID.log" fi - if [ ! -w "$(dirname $LOG_FILE)" ]; then - echo "Cannot write to log [$(dirname $LOG_FILE)]." - else - Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" +else + LOG_FILE="$LOGFILE" +fi +if [ ! -w "$(dirname $LOG_FILE)" ]; then + echo "Cannot write to log [$(dirname $LOG_FILE)]." +else + Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" +fi + +if [ "$IS_STABLE" != "yes" ]; then + Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN" fi - if [ "$IS_STABLE" != "yes" ]; then - Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN" +GetLocalOS +InitLocalOSSettings +PreInit +Init +CheckEnvironment +PostInit +if [ $_QUICK_SYNC -lt 2 ]; then + CheckCurrentConfig +fi +CheckCurrentConfigAll +DATE=$(date) +Logger "-------------------------------------------------------------" "NOTICE" +Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS" +Logger "-------------------------------------------------------------" "NOTICE" +Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" +if [ $sync_on_changes == true ]; then + SyncOnChanges +else + GetRemoteOS + InitRemoteOSSettings + if [ $no_maxtime == true ]; then + SOFT_MAX_EXEC_TIME=0 + HARD_MAX_EXEC_TIME=0 fi - - GetLocalOS - InitLocalOSSettings - PreInit - Init - CheckEnvironment - PostInit - if [ $_QUICK_SYNC -lt 2 ]; then - CheckCurrentConfig + CheckReplicaPaths + CheckDiskSpace + RunBeforeHook + Main + if [ $? == 0 ]; then + SoftDelete fi - CheckCurrentConfigAll - - DATE=$(date) - Logger "-------------------------------------------------------------" "NOTICE" - Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "NOTICE" - Logger "-------------------------------------------------------------" "NOTICE" - Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" - - if [ $sync_on_changes == true ]; then - SyncOnChanges - else - GetRemoteOS - InitRemoteOSSettings - - if [ $no_maxtime == true ]; then - SOFT_MAX_EXEC_TIME=0 - HARD_MAX_EXEC_TIME=0 - fi - CheckReplicaPaths - CheckDiskSpace - RunBeforeHook - Main - if [ $? == 0 ]; then - SoftDelete - fi + if [ $_SUMMARY == true ]; then + Summary fi +fi diff --git a/osync.sh b/osync.sh index a9274d6..3138158 100755 --- a/osync.sh +++ b/osync.sh @@ -4,22 +4,23 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-beta2 -PROGRAM_BUILD=2016111401 +PROGRAM_BUILD=2016111403 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016111401 +## FUNC_BUILD=2016111402 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name ## INSTANCE_ID=program-instance-name ## _DEBUG=yes/no -## _LOGGER_STDERR=True/False -## _LOGGER_ERR_ONLY=True/False +## _LOGGER_STDERR=true/false +## _LOGGER_ERR_ONLY=true/false +## _LOGGER_PREFIX="date"/"time"/"" #TODO: Windows checks, check sendmail & mailsend @@ -150,7 +151,7 @@ function Logger { WARN_ALERT=true return elif [ "$level" == "NOTICE" ]; then - if [ "$_LOGGER_ERR_ONLY" != True ]; then + if [ "$_LOGGER_ERR_ONLY" != true ]; then _Logger "$prefix$value" fi return @@ -159,6 +160,11 @@ function Logger { _Logger "$prefix$value" fi return + elif [ "$level" == "ALWAYS" ]; then + if [ $_SILENT != true ]; then + _Logger "$prefix$value" "$prefix$level:$value" "$level:$value" + fi + return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then _Logger "$prefix$value" @@ -272,9 +278,9 @@ function SendAlert { eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" if [ $? != 0 ]; then Logger "Cannot create [$ALERT_LOG_FILE]" "WARN" - attachment=False + attachment=false else - attachment=True + attachment=true fi body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG" @@ -292,14 +298,14 @@ function SendAlert { subject="Fnished run - $subject" fi - if [ "$attachment" == True ]; then + if [ "$attachment" == true ]; then attachmentFile="$ALERT_LOG_FILE" fi SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$ENCRYPTION" "SMTP_USER" "$SMTP_PASSWORD" # Delete tmp log file - if [ "$attachment" == True ]; then + if [ "$attachment" == true ]; then if [ -f "$ALERT_LOG_FILE" ]; then rm -f "$ALERT_LOG_FILE" fi @@ -489,7 +495,7 @@ function LoadConfigFile { } function Spinner { - if [ $_SILENT == true ]; then + if [ $_SILENT == true ] || [ $_LOGGER_ERR_ONLY == true ]; then return 0 fi @@ -2404,6 +2410,7 @@ function _deleteLocal { Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir]." "VERBOSE" mv -f "$replicaDir$files" "$replicaDir$deletionDir" fi + echo "$replicaDir$files" >> "$RUN_DIR/$PROGRAM.delete.$replicaType.$SCRIPT_PID" if [ $? != 0 ]; then Logger "Cannot move [$replicaDir$files] to deletion directory." "ERROR" echo "$files" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" @@ -3075,6 +3082,41 @@ function SoftDelete { fi } +function _SummaryFromFile { + local summaryFile="${1}" + local direction="${2}" + + + if [ -f "$summaryFile" ]; then + while read -r file; do + Logger "$direction $(echo $file | awk '{for (i=2; i> TARGET" "ALWAYS" + + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>" + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<" + + Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS" + _SummaryFromFile "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID" ">>" + _SummaryFromFile "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID" "<<" + + Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS" + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>" + _SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<" + + _LOGGER_PREFIX="$prefix" +} + function Init { # Set error exit code if a piped command fails @@ -3209,7 +3251,7 @@ function Init { ## Set sync only function arguments for rsync SYNC_OPTS="-u" - if [ $_VERBOSE == true ]; then + if [ $_VERBOSE == true ] || [ $_SUMMARY == true ]; then SYNC_OPTS=$SYNC_OPTS" -i" fi @@ -3268,8 +3310,10 @@ function Usage { echo "" echo "[OPTIONS]" echo "--dry Will run osync without actually doing anything; just testing" + echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--verbose Increases output" echo "--stats Adds rsync transfer statistics to verbose output" echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" @@ -3342,7 +3386,6 @@ fi INITIATOR_LOCK_FILE_EXISTS=false TARGET_LOCK_FILE_EXISTS=false - FORCE_UNLOCK=false no_maxtime=false opts="" @@ -3355,6 +3398,7 @@ _QUICK_SYNC=0 sync_on_changes=false _NOLOCKS=false osync_cmd=$0 +_SUMMARY=false if [ $# -eq 0 ] then @@ -3433,9 +3477,16 @@ for i in "$@"; do --no-locks) _NOLOCKS=true ;; - --only-errors) - _LOGGER_STDERR=True - _LOGGER_ERR_ONLY=True + --errors-only) + #TODO: let err_only only output to stderr + #_LOGGER_STDERR=true + _LOGGER_ERR_ONLY=true + ;; + --summary) + _SUMMARY=true + ;; + --no-prefix) + _LOGGER_PREFIX="" ;; *) if [ $first == "0" ]; then @@ -3489,59 +3540,59 @@ opts="${opts# *}" LoadConfigFile "$ConfigFile" fi - if [ "$LOGFILE" == "" ]; then - if [ -w /var/log ]; then - LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log" - elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then - LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log" - else - LOG_FILE="./$PROGRAM.$INSTANCE_ID.log" - fi +if [ "$LOGFILE" == "" ]; then + if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log" + elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then + LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log" else - LOG_FILE="$LOGFILE" + LOG_FILE="./$PROGRAM.$INSTANCE_ID.log" fi - if [ ! -w "$(dirname $LOG_FILE)" ]; then - echo "Cannot write to log [$(dirname $LOG_FILE)]." - else - Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" +else + LOG_FILE="$LOGFILE" +fi +if [ ! -w "$(dirname $LOG_FILE)" ]; then + echo "Cannot write to log [$(dirname $LOG_FILE)]." +else + Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" +fi + +if [ "$IS_STABLE" != "yes" ]; then + Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN" fi - if [ "$IS_STABLE" != "yes" ]; then - Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN" +GetLocalOS +InitLocalOSSettings +PreInit +Init +CheckEnvironment +PostInit +if [ $_QUICK_SYNC -lt 2 ]; then + CheckCurrentConfig +fi +CheckCurrentConfigAll +DATE=$(date) +Logger "-------------------------------------------------------------" "NOTICE" +Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS" +Logger "-------------------------------------------------------------" "NOTICE" +Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" +if [ $sync_on_changes == true ]; then + SyncOnChanges +else + GetRemoteOS + InitRemoteOSSettings + if [ $no_maxtime == true ]; then + SOFT_MAX_EXEC_TIME=0 + HARD_MAX_EXEC_TIME=0 fi - - GetLocalOS - InitLocalOSSettings - PreInit - Init - CheckEnvironment - PostInit - if [ $_QUICK_SYNC -lt 2 ]; then - CheckCurrentConfig + CheckReplicaPaths + CheckDiskSpace + RunBeforeHook + Main + if [ $? == 0 ]; then + SoftDelete fi - CheckCurrentConfigAll - - DATE=$(date) - Logger "-------------------------------------------------------------" "NOTICE" - Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "NOTICE" - Logger "-------------------------------------------------------------" "NOTICE" - Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" - - if [ $sync_on_changes == true ]; then - SyncOnChanges - else - GetRemoteOS - InitRemoteOSSettings - - if [ $no_maxtime == true ]; then - SOFT_MAX_EXEC_TIME=0 - HARD_MAX_EXEC_TIME=0 - fi - CheckReplicaPaths - CheckDiskSpace - RunBeforeHook - Main - if [ $? == 0 ]; then - SoftDelete - fi + if [ $_SUMMARY == true ]; then + Summary fi +fi