From 97a1aa5e056059a03133e022561f2e68ae557e66 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 19 Dec 2016 13:19:21 +0100 Subject: [PATCH 01/16] Updated sudo behavior --- dev/n_osync.sh | 55 ++++++++++++++++++++++++----------------------- dev/ofunctions.sh | 4 ++-- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index d54e103..634853a 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-RC1+dev -PROGRAM_BUILD=2016121502 +PROGRAM_BUILD=2016121901 IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG @@ -210,7 +210,7 @@ function _CheckReplicasLocal { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval @@ -261,16 +261,17 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### IsInteger SUBSET #### include #### HumanToNumeric SUBSET #### include #### RemoteLogger SUBSET #### + function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -370,7 +371,7 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -412,7 +413,7 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -445,7 +446,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### ArrayContains SUBSET #### @@ -454,7 +455,7 @@ include #### RemoteLogger SUBSET #### function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -506,7 +507,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -589,7 +590,7 @@ function _UnlockReplicasLocal { local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then Logger "Could not unlock local $replicaType replica." "ERROR" $retval @@ -613,9 +614,9 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? @@ -817,7 +818,7 @@ function _getFileCtimeMtimeRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -1056,7 +1057,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -1159,9 +1160,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### RemoteLogger SUBSET #### @@ -1174,7 +1175,7 @@ include #### RemoteLogger SUBSET #### previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -1189,7 +1190,7 @@ include #### RemoteLogger SUBSET #### if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -1197,11 +1198,11 @@ include #### RemoteLogger SUBSET #### parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -1217,7 +1218,7 @@ include #### RemoteLogger SUBSET #### if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -1667,7 +1668,7 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval @@ -1676,7 +1677,7 @@ function _SoftDeleteLocal { Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval @@ -1720,13 +1721,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 4f4d8be..3bbc3f2 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,7 +3,7 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016121501 +_OFUNCTIONS_BUILD=2016121901 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -1556,7 +1556,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" From e04e81446e1d1ca0b7524f3d0f4fc37ecd6d9253 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 19 Dec 2016 22:38:13 +0100 Subject: [PATCH 02/16] Fixed typo --- dev/merge.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/merge.sh b/dev/merge.sh index 93138b2..fbc8d74 100755 --- a/dev/merge.sh +++ b/dev/merge.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -## MERGE 2016121101 +## MERGE 2016121901 ## Merges ofunctions.sh and n_program.sh into program.sh ## Adds installer function __PREPROCESSOR_Merge { - PROGRAM=osync + PROGRAM=pmocr VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh) VERSION=${VERSION#*=} @@ -40,7 +40,7 @@ function __PREPROCESSOR_Constants { __PREPROCESSOR_SUBSETS=( '#### OFUNCTIONS FULL SUBSET ####' - '#### OFUNCTIONS MINIMAL SUBSET ####' + '#### OFUNCTIONS MINI SUBSET ####' '#### _OFUNCTIONS_BOOTSTRAP SUBSET ####' '#### DEBUG SUBSET ####' '#### TrapError SUBSET ####' From 266b35d81bfdb4499e86153416fdcd2a53ff5006 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 19 Dec 2016 22:39:03 +0100 Subject: [PATCH 03/16] Fixing typos --- dev/ofunctions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 3bbc3f2..83ed597 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,7 +3,7 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016121901 +_OFUNCTIONS_BUILD=2016121902 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -202,7 +202,7 @@ function RemoteLogger { # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr -# _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels From ea69a58b3d01556058410154829cddb1f7ed8583 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 19 Dec 2016 23:40:54 +0100 Subject: [PATCH 04/16] Fixed program name after fix from other project --- dev/merge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/merge.sh b/dev/merge.sh index fbc8d74..c2476ed 100755 --- a/dev/merge.sh +++ b/dev/merge.sh @@ -6,7 +6,7 @@ ## Adds installer function __PREPROCESSOR_Merge { - PROGRAM=pmocr + PROGRAM=osync VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh) VERSION=${VERSION#*=} From 79402ef2dad8063e54e3d09b1171ca8fef3baaf9 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 23 Dec 2016 18:25:08 +0100 Subject: [PATCH 05/16] Updated ofunctions from obackup --- dev/ofunctions.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 83ed597..5e2fe1e 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,7 +3,7 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016121902 +_OFUNCTIONS_BUILD=2016122302 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -113,6 +113,13 @@ function Dummy { #### Logger SUBSET #### #### RemoteLogger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -192,8 +199,8 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } #### RemoteLogger SUBSET END #### @@ -253,7 +260,7 @@ function Logger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "$prefix$value" "$prefix$value" + _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -266,8 +273,8 @@ function Logger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### @@ -617,11 +624,6 @@ function Spinner { fi } -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 From f4126df723bc9ad31b04a3ef2abc5f6bd1aef06e Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 23 Dec 2016 20:41:23 +0100 Subject: [PATCH 06/16] Enable remote host ping on local tests --- dev/tests/run_tests.sh | 82 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 42e6fff..123af87 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -7,7 +7,7 @@ ## On CYGWIN / MSYS, ACL and extended attributes aren't supported -# osync test suite 2016121301 +# osync test suite 2016122301 # 4 tests: # quicklocal @@ -199,10 +199,12 @@ function oneTimeSetUp () { if [ "$TRAVIS_RUN" == true ]; then echo "Running with travis settings" REMOTE_USER="travis" + RHOST_PING="no" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" else echo "Running with local settings" REMOTE_USER="root" + RHOST_PING="yes" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" fi @@ -311,7 +313,7 @@ function test_LargeFileSet () { PrepareLocalDirs DownloadLargeFileSet "$INITIATOR_DIR" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "LargeFileSet test with parameters [$i]." "0" $? [ -d "$INITIATOR_DIR/$OSYNC_STATE_DIR" ] @@ -339,7 +341,7 @@ function test_Exclusions () { numberOfPHPFiles=$(find "$INITIATOR_DIR" ! -wholename "$INITIATOR_DIR/$OSYNC_WORKDIR*" -name "*.php" | wc -l) - REMOTE_HOST_PING=no RSYNC_EXCLUDE_PATTERN="*.php" ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING RSYNC_EXCLUDE_PATTERN="*.php" ./$OSYNC_EXECUTABLE $i assertEquals "Exclusions with parameters [$i]." "0" $? numberOfInitiatorFiles=$(find "$INITIATOR_DIR" ! -wholename "$INITIATOR_DIR/$OSYNC_WORKDIR*" | wc -l) @@ -366,13 +368,13 @@ function test_Deletetion () { touch "$tFile1" touch "$tFile2" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? rm -f "$iFile1" rm -f "$tFile1" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second deletion run with parameters [$i]." "0" $? [ -f "$TARGET_DIR/$OSYNC_DELETE_DIR/$(basename $iFile1)" ] @@ -417,7 +419,7 @@ function test_deletion_failure () { touch "$INITIATOR_DIR/$FileA" touch "$TARGET_DIR/$FileB" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? rm -f "$INITIATOR_DIR/$FileA" @@ -428,7 +430,7 @@ function test_deletion_failure () { $SUDO_CMD $IMMUTABLE_ON_CMD "$INITIATOR_DIR/$FileB" # This shuold fail with exitcode 1 - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second deletion run with parameters [$i]." "1" $? # standard file tests @@ -446,7 +448,7 @@ function test_deletion_failure () { $SUDO_CMD $IMMUTABLE_OFF_CMD "$TARGET_DIR/$FileA" $SUDO_CMD $IMMUTABLE_OFF_CMD "$INITIATOR_DIR/$FileB" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Third deletion run with parameters [$i]." "0" $? [ ! -f "$TARGET_DIR/$FileA" ] @@ -498,14 +500,14 @@ function test_skip_deletion () { touch "$TARGET_DIR/$FileB" # First run - REMOTE_HOST_PING=no SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? rm -f "$INITIATOR_DIR/$FileA" rm -f "$TARGET_DIR/$FileB" # Second run - REMOTE_HOST_PING=no SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? if [ "$mode" == "initiator" ]; then @@ -578,14 +580,14 @@ function test_handle_symlinks () { ln -s "$INITIATOR_DIR/$FileA" "$INITIATOR_DIR/$FileAL" ln -s "$TARGET_DIR/$FileB" "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First symlink run with parameters [$i]." "0" $? # Delete symlinks rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second symlink deletion run with parameters [$i]." "0" $? # symlink deletion propagation @@ -604,7 +606,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileA" rm -f "$TARGET_DIR/$FileB" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Third broken symlink run with parameters [$i]." "0" $? [ -L "$TARGET_DIR/$FileAL" ] @@ -617,7 +619,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Fourth symlink deletion run with parameters [$i]." "0" $? [ ! -L "$TARGET_DIR/$FileAL" ] @@ -665,14 +667,14 @@ function test_handle_symlinks () { ln -s "$INITIATOR_DIR/$FileA" "$INITIATOR_DIR/$FileAL" ln -s "$TARGET_DIR/$FileB" "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First symlink run with parameters [$i]." "0" $? # Delete symlinks rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second symlink deletion run with parameters [$i]." "0" $? # symlink deletion propagation @@ -691,7 +693,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileA" rm -f "$TARGET_DIR/$FileB" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Third broken symlink run with parameters should fail [$i]." "1" $? [ ! -f "$TARGET_DIR/$FileAL" ] @@ -704,7 +706,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Fourth symlink deletion run should resume with parameters [$i]." "0" $? [ ! -f "$TARGET_DIR/$FileAL" ] @@ -735,7 +737,7 @@ function test_softdeletion_cleanup () { PrepareLocalDirs # First run - #REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + #REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i #assertEquals "First deletion run with parameters [$i]." "0" $? # Get current drive @@ -764,7 +766,7 @@ function test_softdeletion_cleanup () { fi # Second run - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i # Check file presence for file in "${files[@]}"; do @@ -828,7 +830,7 @@ function test_FileAttributePropagation () { touch "$TARGET_DIR/$FileB" # First run - PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? sleep 1 @@ -856,7 +858,7 @@ function test_FileAttributePropagation () { assertEquals "Set ACL on target directory" "0" $? # Second run - PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? getfacl "$TARGET_DIR/$FileA" | grep "other::r-x" > /dev/null @@ -891,14 +893,14 @@ function test_ConflictBackups () { echo "$FileB" > "$TARGET_DIR/$FileB" # First run - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA+" > "$TARGET_DIR/$FileA" echo "$FileB+" > "$INITIATOR_DIR/$FileB" # Second run - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? [ -f "$INITIATOR_DIR/$OSYNC_BACKUP_DIR/$FileA" ] @@ -935,28 +937,28 @@ function test_MultipleConflictBackups () { echo "$FileB" > "$TARGET_DIR/$FileB" # First run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA+" > "$TARGET_DIR/$FileA" echo "$FileB+" > "$INITIATOR_DIR/$FileB" # Second run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA-" > "$TARGET_DIR/$FileA" echo "$FileB-" > "$INITIATOR_DIR/$FileB" # Third run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA*" > "$TARGET_DIR/$FileA" echo "$FileB*" > "$INITIATOR_DIR/$FileB" # Fouth run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? # This test may fail only on 31th December at 23:59 :) @@ -986,12 +988,12 @@ function test_Locking () { mkdir -p "$INITIATOR_DIR/$OSYNC_WORKDIR" echo 65536 > "$INITIATOR_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Should be able to resume when initiator has lock without running pid." "0" $? echo $$ > "$INITIATOR_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Should never be able to resume when initiator has lock with running pid." "1" $? done @@ -1000,14 +1002,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@quicklocal > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} assertEquals "Should be able to resume locked target with same instance_id in quickLocal mode." "0" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@local > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} assertEquals "Should be able to resume locked target with same instance_id in confLocal mode." "0" $? if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ]; then @@ -1015,14 +1017,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@quickremote > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} assertEquals "Should be able to resume locked target with same instance_id in quickRemote mode." "0" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@remote > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} assertEquals "Should be able to resume locked target with same instance_id in confRemote mode." "0" $? fi @@ -1031,14 +1033,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} assertEquals "Should be able to resume locked local target with bogus instance id in quickLocal mode." "0" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} assertEquals "Should be able to resume locked local target with bogus instance_id in confLocal mode." "0" $? if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ]; then @@ -1046,14 +1048,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} assertEquals "Should not be able to resume remote locked target with bogus instance_id in quickRemote mode." "1" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} assertEquals "Should not be able to resume remote locked target with bogus instance_id in confRemote mode." "1" $? fi @@ -1070,7 +1072,7 @@ function test_Locking () { mkdir -p "$INITIATOR_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$INITIATOR_DIR/$OSYNC_WORKDIR/lock" - FORCE_STRANGER_UNLOCK=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + FORCE_STRANGER_UNLOCK=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Should be able to resume when target has lock with different instance id but FORCE_STRANGER_UNLOCK=yes." "0" $? done @@ -1363,7 +1365,7 @@ function test_NoRemoteAccessTest () { cd "$OSYNC_DIR" PrepareLocalDirs - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} assertEquals "Basic local test without remote access." "0" $? } From 07daa1351767a2073d9caa311168a81cac7d0b12 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 23 Dec 2016 20:41:52 +0100 Subject: [PATCH 07/16] Some minor fixes from obackup --- dev/ofunctions.sh | 93 ++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 42 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 5e2fe1e..fc35e97 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,7 +3,7 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016122302 +_OFUNCTIONS_BUILD=2016122303 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -1594,6 +1594,50 @@ function PostInit { fi } +function SetCompression { + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + function InitLocalOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1628,8 +1672,12 @@ function InitLocalOSDependingSettings { STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi + + # Set compression first time when we know what local os we have + SetCompression } +# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1722,47 +1770,8 @@ function InitRemoteOSDependingSettings { RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + # Set compression options again after we know what remote OS we're dealing with + SetCompression } ## IFS debug function From 9fd881647a28d4a5b950c9210393053f49316df0 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 23 Dec 2016 21:25:04 +0100 Subject: [PATCH 08/16] Enabled remote host ping when not testing on travis --- dev/tests/run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 123af87..838afd0 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -7,7 +7,7 @@ ## On CYGWIN / MSYS, ACL and extended attributes aren't supported -# osync test suite 2016122301 +# osync test suite 2016122302 # 4 tests: # quicklocal @@ -201,11 +201,13 @@ function oneTimeSetUp () { REMOTE_USER="travis" RHOST_PING="no" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" + SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" "no" else echo "Running with local settings" REMOTE_USER="root" RHOST_PING="yes" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" + SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" "yes" fi # Get default ssh port from env From 1e0fb77cc894b2bb696ee8e119aa4e090173234d Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 23 Dec 2016 21:30:38 +0100 Subject: [PATCH 09/16] Fixed possible issue with old file upgrade test when file is intended for remote cnx --- dev/tests/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 838afd0..af4be41 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -1299,6 +1299,10 @@ function test_UpgradeConfRun () { ./$OSYNC_UPGRADE "$CONF_DIR/$TMP_OLD_CONF" assertEquals "Conf file upgrade" "0" $? + + # Update remote conf files with SSH port + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${HOME}/osync-tests/target#' "$CONF_DIR/$TMP_OLD_CONF" + ./$OSYNC_EXECUTABLE "$CONF_DIR/$TMP_OLD_CONF" assertEquals "Upgraded conf file execution test" "0" $? From 2ec41074ae60b114100fcfdd8bbf4e244da7a452 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 23 Dec 2016 21:37:06 +0100 Subject: [PATCH 10/16] More unneeded fixes for old upgrade remote connections --- dev/tests/run_tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index af4be41..68143fc 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -202,12 +202,18 @@ function oneTimeSetUp () { RHOST_PING="no" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" "no" + + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOSTS" "" + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "no" else echo "Running with local settings" REMOTE_USER="root" RHOST_PING="yes" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" "yes" + + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "yes" fi # Get default ssh port from env From 4c6ada238874613d978921beb362f6e0b328f5d0 Mon Sep 17 00:00:00 2001 From: deajan Date: Wed, 28 Dec 2016 23:37:07 +0100 Subject: [PATCH 11/16] Added small fixes from obackup --- dev/common_install.sh | 6 ++++-- dev/ofunctions.sh | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dev/common_install.sh b/dev/common_install.sh index 3d8dd77..d5d024e 100755 --- a/dev/common_install.sh +++ b/dev/common_install.sh @@ -6,7 +6,7 @@ PROGRAM=[prgname] PROGRAM_VERSION=[version] PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" -SCRIPT_BUILD=2016121301 +SCRIPT_BUILD=2016122701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -48,6 +48,8 @@ include #### GetLocalOS SUBSET #### function SetLocalOSSettings { USER=root + # LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS + case $LOCAL_OS in *"BSD"*) GROUP=wheel @@ -74,7 +76,7 @@ function SetLocalOSSettings { exit 1 fi - OS=$(UrlEncode "$localOsVar") + OS=$(UrlEncode "$LOCAL_OS_FULL") } function GetInit { diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index fc35e97..be2287e 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,7 +3,7 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016122303 +_OFUNCTIONS_BUILD=2016122701 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -150,7 +150,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -1098,7 +1098,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1148,6 +1148,9 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } #### GetLocalOS SUBSET END #### @@ -1175,7 +1178,7 @@ function GetOs { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then From 9e5a2c82f10073a536463da639ebe62af154cdac Mon Sep 17 00:00:00 2001 From: deajan Date: Wed, 28 Dec 2016 23:37:29 +0100 Subject: [PATCH 12/16] Rebuilt targets --- dev/debug_osync.sh | 318 ++++++++++++++++++++++++++------------------- install.sh | 13 +- osync.sh | 310 ++++++++++++++++++++++++------------------- 3 files changed, 367 insertions(+), 274 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index f60b139..d8f4f6a 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -3,9 +3,9 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-RC1 -PROGRAM_BUILD=2016121302 -IS_STABLE=yes +PROGRAM_VERSION=1.2-RC1+dev +PROGRAM_BUILD=2016121901 +IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -40,10 +40,9 @@ IS_STABLE=yes # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -#### OFUNCTIONS MINI SUBSET #### -_OFUNCTIONS_VERSION=2.1-RC1 -_OFUNCTIONS_BUILD=2016121304 +_OFUNCTIONS_VERSION=2.1-RC1+dev +_OFUNCTIONS_BUILD=2016122701 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -148,6 +147,13 @@ function Dummy { } #### Logger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -178,7 +184,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -186,7 +192,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -227,8 +233,8 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -236,7 +242,7 @@ function RemoteLogger { # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr -# _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels @@ -261,7 +267,7 @@ function Logger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "$prefix($level):$value" "$prefix\e[41m$value\e[0m" true + _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true # ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag. echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" @@ -287,7 +293,7 @@ function Logger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "$prefix$value" "$prefix$value" + _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -300,8 +306,8 @@ function Logger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### @@ -647,11 +653,6 @@ function Spinner { fi } -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -1117,7 +1118,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1167,9 +1168,11 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } -#### OFUNCTIONS MINI SUBSET END #### function GetRemoteOS { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1193,7 +1196,7 @@ function GetOs { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1576,7 +1579,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" @@ -1612,6 +1615,50 @@ function PostInit { fi } +function SetCompression { + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + function InitLocalOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1646,8 +1693,12 @@ function InitLocalOSDependingSettings { STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi + + # Set compression first time when we know what local os we have + SetCompression } +# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1740,47 +1791,8 @@ function InitRemoteOSDependingSettings { RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + # Set compression options again after we know what remote OS we're dealing with + SetCompression } ## IFS debug function @@ -1961,24 +1973,26 @@ function CheckCurrentConfigAll { function _CheckReplicasLocal { local replicaPath="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local diskSpace if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + Logger "Local replica path [$replicaPath] does not exist / is not writable." "CRITICAL" return 1 fi fi @@ -2007,7 +2021,9 @@ function _CheckReplicasLocal { function _CheckReplicasRemote { local replicaPath="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -2017,7 +2033,7 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -2083,6 +2099,13 @@ function HumanToNumeric { echo $value } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -2113,7 +2136,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -2121,7 +2144,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2162,15 +2185,15 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } -function _CheckReplicasRemoteSub { +function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -2179,7 +2202,7 @@ function _CheckReplicasRemoteSub { RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" fi else - RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + RemoteLogger "Remote replica path [$replicaPath] does not exist / is not writable." "CRITICAL" exit 1 fi fi @@ -2212,10 +2235,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Failed to check remote replica." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2238,13 +2261,13 @@ function CheckReplicas { fi fi - _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" & + _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" & pids="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckReplicasLocal "${TARGET[$__replicaDir]}" & + _CheckReplicasLocal "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _CheckReplicasRemote "${TARGET[$__replicaDir]}" & + _CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" fi WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false @@ -2270,11 +2293,11 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 fi fi @@ -2312,11 +2335,11 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval - Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" @@ -2345,7 +2368,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -2399,6 +2422,13 @@ function IsInteger { echo 0 fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -2429,7 +2459,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -2437,7 +2467,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2478,14 +2508,14 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -2537,7 +2567,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -2555,10 +2585,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Remote lock handling failed." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2613,24 +2643,28 @@ function HandleLocks { function _UnlockReplicasLocal { local lockfile="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock local replica." "ERROR" $retval + Logger "Could not unlock local $replicaType replica." "ERROR" $retval else - Logger "Removed local replica lock." "DEBUG" + Logger "Removed local $replicaType replica lock." "DEBUG" fi fi } function _UnlockReplicasRemote { local lockfile="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -2640,17 +2674,17 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock remote replica." "ERROR" $retval - Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Could not unlock $replicaType remote replica." "ERROR" $retval + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else - Logger "Removed remote replica lock." "DEBUG" + Logger "Removed remote $replicaType replica lock." "DEBUG" fi } @@ -2664,16 +2698,16 @@ function UnlockReplicas { fi if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then - _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" & + _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" & pids="$!" fi if [ $TARGET_LOCK_FILE_EXISTS == true ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then - _UnlockReplicasLocal "${TARGET[$__lockFile]}" & + _UnlockReplicasLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _UnlockReplicasRemote "${TARGET[$__lockFile]}" & + _UnlockReplicasRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" fi fi @@ -2733,10 +2767,11 @@ function treeList { return $? elif [ $retval -eq 23 ]; then Logger "Some files could not be listed in $replicaType replica [$replicaPath]. Check for failing symlinks." "ERROR" $retval - Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return 0 else Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return $retval fi } @@ -2744,6 +2779,7 @@ function treeList { # deleteList(replicaType): Creates a list of files vanished from last run on replica $1 (initiator/target) function deleteList { local replicaType="${1}" # replica type: initiator, target + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -2801,6 +2837,7 @@ function _getFileCtimeMtimeLocal { local replicaPath="${1}" # Contains replica path local replicaType="${2}" # Initiator / Target local fileList="${3}" # Contains list of files to get time attrs + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -2811,7 +2848,7 @@ function _getFileCtimeMtimeLocal { if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi @@ -2834,14 +2871,14 @@ function _getFileCtimeMtimeRemote { if [ $retval -ne 0 ]; then Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -2851,7 +2888,7 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return $retval else @@ -3080,7 +3117,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -3183,9 +3220,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -3212,6 +3249,13 @@ function TrapError { echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -3242,7 +3286,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -3250,7 +3294,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3291,8 +3335,8 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -3304,7 +3348,7 @@ function RemoteLogger { previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -3319,7 +3363,7 @@ function RemoteLogger { if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -3327,11 +3371,11 @@ function RemoteLogger { parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -3347,7 +3391,7 @@ function RemoteLogger { if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -3797,25 +3841,25 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "Directory cleanup complete on $replicaType replica." "NOTICE" fi - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" - elif [ -d "$replicaDeletionPath" ] && ! [ -w "$replicaDeletionPath" ]; then Logger "The $replicaType replica dir [$replicaDeletionPath] is not writable. Cannot clean old files." "ERROR" @@ -3850,13 +3894,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" @@ -3866,7 +3910,7 @@ ENDSSH retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else Logger "Cleanup complete on $replicaType replica." "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" diff --git a/install.sh b/install.sh index b30dc11..c67c1e3 100755 --- a/install.sh +++ b/install.sh @@ -3,10 +3,10 @@ _OFUNCTIONS_BOOTSTRAP=true PROGRAM=osync -PROGRAM_VERSION=1.2-RC1 +PROGRAM_VERSION=1.2-RC1+dev PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" -SCRIPT_BUILD=2016121301 +SCRIPT_BUILD=2016122701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -92,7 +92,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -142,10 +142,15 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } function SetLocalOSSettings { USER=root + # LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS + case $LOCAL_OS in *"BSD"*) GROUP=wheel @@ -172,7 +177,7 @@ function SetLocalOSSettings { exit 1 fi - OS=$(UrlEncode "$localOsVar") + OS=$(UrlEncode "$LOCAL_OS_FULL") } function GetInit { diff --git a/osync.sh b/osync.sh index 10776a5..3b2ae47 100755 --- a/osync.sh +++ b/osync.sh @@ -3,16 +3,15 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-RC1 -PROGRAM_BUILD=2016121302 -IS_STABLE=yes +PROGRAM_VERSION=1.2-RC1+dev +PROGRAM_BUILD=2016121901 +IS_STABLE=no -#### OFUNCTIONS MINI SUBSET #### -_OFUNCTIONS_VERSION=2.1-RC1 -_OFUNCTIONS_BUILD=2016121304 +_OFUNCTIONS_VERSION=2.1-RC1+dev +_OFUNCTIONS_BUILD=2016122701 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -112,6 +111,13 @@ function Dummy { } #### Logger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -142,7 +148,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -150,7 +156,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -186,8 +192,8 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -195,7 +201,7 @@ function RemoteLogger { # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr -# _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels @@ -220,7 +226,7 @@ function Logger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "$prefix($level):$value" "$prefix\e[41m$value\e[0m" true + _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true # ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag. echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" @@ -246,7 +252,7 @@ function Logger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "$prefix$value" "$prefix$value" + _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -254,8 +260,8 @@ function Logger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### @@ -596,11 +602,6 @@ function Spinner { fi } -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -1051,7 +1052,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1101,9 +1102,11 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } -#### OFUNCTIONS MINI SUBSET END #### function GetRemoteOS { @@ -1126,7 +1129,7 @@ function GetOs { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1440,7 +1443,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" @@ -1475,6 +1478,50 @@ function PostInit { fi } +function SetCompression { + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + function InitLocalOSDependingSettings { ## If running under Msys, some commands do not run the same way @@ -1508,8 +1555,12 @@ function InitLocalOSDependingSettings { STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi + + # Set compression first time when we know what local os we have + SetCompression } +# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then @@ -1601,47 +1652,8 @@ function InitRemoteOSDependingSettings { RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + # Set compression options again after we know what remote OS we're dealing with + SetCompression } ## IFS debug function @@ -1819,23 +1831,25 @@ function CheckCurrentConfigAll { function _CheckReplicasLocal { local replicaPath="${1}" + local replicaType="${2}" + local retval local diskSpace if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + Logger "Local replica path [$replicaPath] does not exist / is not writable." "CRITICAL" return 1 fi fi @@ -1864,6 +1878,8 @@ function _CheckReplicasLocal { function _CheckReplicasRemote { local replicaPath="${1}" + local replicaType="${2}" + local retval local cmd @@ -1873,7 +1889,7 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then @@ -1935,6 +1951,13 @@ function HumanToNumeric { echo $value } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -1965,7 +1988,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -1973,7 +1996,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2009,15 +2032,15 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } -function _CheckReplicasRemoteSub { +function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -2026,7 +2049,7 @@ function _CheckReplicasRemoteSub { RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" fi else - RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + RemoteLogger "Remote replica path [$replicaPath] does not exist / is not writable." "CRITICAL" exit 1 fi fi @@ -2059,10 +2082,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Failed to check remote replica." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2084,13 +2107,13 @@ function CheckReplicas { fi fi - _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" & + _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" & pids="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckReplicasLocal "${TARGET[$__replicaDir]}" & + _CheckReplicasLocal "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _CheckReplicasRemote "${TARGET[$__replicaDir]}" & + _CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" fi WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false @@ -2115,11 +2138,11 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 fi fi @@ -2157,11 +2180,11 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval - Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" @@ -2189,7 +2212,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then @@ -2239,6 +2262,13 @@ function IsInteger { echo 0 fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -2269,7 +2299,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -2277,7 +2307,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2313,14 +2343,14 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -2372,7 +2402,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -2390,10 +2420,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Remote lock handling failed." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2447,22 +2477,26 @@ function HandleLocks { function _UnlockReplicasLocal { local lockfile="${1}" + local replicaType="${2}" + local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock local replica." "ERROR" $retval + Logger "Could not unlock local $replicaType replica." "ERROR" $retval else - Logger "Removed local replica lock." "DEBUG" + Logger "Removed local $replicaType replica lock." "DEBUG" fi fi } function _UnlockReplicasRemote { local lockfile="${1}" + local replicaType="${2}" + local retval local cmd @@ -2472,17 +2506,17 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock remote replica." "ERROR" $retval - Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Could not unlock $replicaType remote replica." "ERROR" $retval + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else - Logger "Removed remote replica lock." "DEBUG" + Logger "Removed remote $replicaType replica lock." "DEBUG" fi } @@ -2495,16 +2529,16 @@ function UnlockReplicas { fi if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then - _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" & + _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" & pids="$!" fi if [ $TARGET_LOCK_FILE_EXISTS == true ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then - _UnlockReplicasLocal "${TARGET[$__lockFile]}" & + _UnlockReplicasLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _UnlockReplicasRemote "${TARGET[$__lockFile]}" & + _UnlockReplicasRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" fi fi @@ -2563,10 +2597,11 @@ function treeList { return $? elif [ $retval -eq 23 ]; then Logger "Some files could not be listed in $replicaType replica [$replicaPath]. Check for failing symlinks." "ERROR" $retval - Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return 0 else Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return $retval fi } @@ -2575,6 +2610,7 @@ function treeList { function deleteList { local replicaType="${1}" # replica type: initiator, target + local retval local subretval local cmd @@ -2631,6 +2667,7 @@ function _getFileCtimeMtimeLocal { local replicaType="${2}" # Initiator / Target local fileList="${3}" # Contains list of files to get time attrs + local retval echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -2639,7 +2676,7 @@ function _getFileCtimeMtimeLocal { if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi @@ -2661,14 +2698,14 @@ function _getFileCtimeMtimeRemote { if [ $retval -ne 0 ]; then Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -2678,7 +2715,7 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return $retval else @@ -2904,7 +2941,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -3006,9 +3043,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then @@ -3031,6 +3068,13 @@ function TrapError { echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -3061,7 +3105,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -3069,7 +3113,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3105,8 +3149,8 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -3118,7 +3162,7 @@ function RemoteLogger { previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -3133,7 +3177,7 @@ function RemoteLogger { if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -3141,11 +3185,11 @@ function RemoteLogger { parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -3161,7 +3205,7 @@ function RemoteLogger { if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -3608,25 +3652,25 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "Directory cleanup complete on $replicaType replica." "NOTICE" fi - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" - elif [ -d "$replicaDeletionPath" ] && ! [ -w "$replicaDeletionPath" ]; then Logger "The $replicaType replica dir [$replicaDeletionPath] is not writable. Cannot clean old files." "ERROR" @@ -3660,13 +3704,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" @@ -3676,7 +3720,7 @@ ENDSSH retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else Logger "Cleanup complete on $replicaType replica." "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" From 6fc4bef2240245f4ca125fb7b3c8449f3baac8f0 Mon Sep 17 00:00:00 2001 From: deajan Date: Wed, 4 Jan 2017 09:17:49 +0100 Subject: [PATCH 13/16] Updated ofunctions from obackup --- dev/ofunctions.sh | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index be2287e..f2b8dbd 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,11 +3,12 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016122701 +_OFUNCTIONS_BUILD=2017010401 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### -## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name @@ -586,7 +587,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } #### TrapError SUBSET END #### @@ -680,7 +681,7 @@ function WaitForTaskCompletion { Spinner fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -836,7 +837,7 @@ function ParallelExec { fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -874,7 +875,7 @@ function ParallelExec { fi eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) + return $((commandCount - counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -897,7 +898,6 @@ function ParallelExec { if [ $(IsInteger $pid) -eq 1 ]; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) if kill -0 $pid > /dev/null 2>&1; then - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -921,7 +921,7 @@ function ParallelExec { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done return $errorCount @@ -937,11 +937,6 @@ function CleanUp { fi } -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1423,8 +1418,8 @@ function __CheckArguments { if [ "$argument" == "" ]; then fetchArguments=false else - argList="$argList[Argument $(($iterate-2)): $argument] " - iterate=$(($iterate+1)) + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) fi done @@ -1440,7 +1435,7 @@ function __CheckArguments { Logger "Entering function [$callerName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" Logger "$callerName arguments: $argList" "ERROR" else if [ ! -z "$argList" ]; then From e1ea7806cb7de6ed0896922a10c971291166d4f2 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 3 Feb 2017 09:51:52 +0100 Subject: [PATCH 14/16] Rebuilt targets --- dev/debug_osync.sh | 35 +++++++++++++++-------------------- osync.sh | 29 ++++++++++++----------------- 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index d8f4f6a..0b6858a 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2016 by Orsiris de Jong" +AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-RC1+dev PROGRAM_BUILD=2016121901 @@ -42,9 +42,10 @@ IS_STABLE=no _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016122701 +_OFUNCTIONS_BUILD=2017010401 _OFUNCTIONS_BOOTSTRAP=true -## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name @@ -616,7 +617,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } @@ -709,7 +710,7 @@ function WaitForTaskCompletion { Spinner fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -865,7 +866,7 @@ function ParallelExec { fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -903,7 +904,7 @@ function ParallelExec { fi eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) + return $((commandCount - counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -926,7 +927,6 @@ function ParallelExec { if [ $(IsInteger $pid) -eq 1 ]; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) if kill -0 $pid > /dev/null 2>&1; then - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -950,7 +950,7 @@ function ParallelExec { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done return $errorCount @@ -966,11 +966,6 @@ function CleanUp { fi } -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1441,8 +1436,8 @@ function __CheckArguments { if [ "$argument" == "" ]; then fetchArguments=false else - argList="$argList[Argument $(($iterate-2)): $argument] " - iterate=$(($iterate+1)) + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) fi done @@ -1458,7 +1453,7 @@ function __CheckArguments { Logger "Entering function [$callerName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" Logger "$callerName arguments: $argList" "ERROR" else if [ ! -z "$argList" ]; then @@ -2057,7 +2052,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } function IsInteger { @@ -2392,7 +2387,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } ## Modified version of http://stackoverflow.com/a/8574392 @@ -3246,7 +3241,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } diff --git a/osync.sh b/osync.sh index 3b2ae47..a23cfd3 100755 --- a/osync.sh +++ b/osync.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2016 by Orsiris de Jong" +AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-RC1+dev PROGRAM_BUILD=2016121901 @@ -11,9 +11,10 @@ IS_STABLE=no _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016122701 +_OFUNCTIONS_BUILD=2017010401 _OFUNCTIONS_BOOTSTRAP=true -## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name @@ -566,7 +567,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } @@ -655,7 +656,7 @@ function WaitForTaskCompletion { Spinner fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -804,7 +805,7 @@ function ParallelExec { fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -842,7 +843,7 @@ function ParallelExec { fi eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) + return $((commandCount - counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -865,7 +866,6 @@ function ParallelExec { if [ $(IsInteger $pid) -eq 1 ]; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) if kill -0 $pid > /dev/null 2>&1; then - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -885,7 +885,7 @@ function ParallelExec { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done return $errorCount @@ -900,11 +900,6 @@ function CleanUp { fi } -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1909,7 +1904,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } function IsInteger { @@ -2232,7 +2227,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } ## Modified version of http://stackoverflow.com/a/8574392 @@ -3065,7 +3060,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } From d1e5a50c0d1c0f6d4d5f78244c4f5510ff48c7e8 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 3 Feb 2017 09:52:24 +0100 Subject: [PATCH 15/16] Update files --- dev/n_osync.sh | 6 +++--- dev/tests/conf/remote.conf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 634853a..ca13e97 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2016 by Orsiris de Jong" +AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-RC1+dev PROGRAM_BUILD=2016121901 @@ -220,7 +220,7 @@ function _CheckReplicasLocal { Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + Logger "Local replica path [$replicaPath] does not exist / is not writable." "CRITICAL" return 1 fi fi @@ -280,7 +280,7 @@ function _CheckReplicasRemoteSub { RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" fi else - RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + RemoteLogger "Remote replica path [$replicaPath] does not exist / is not writable." "CRITICAL" exit 1 fi fi diff --git a/dev/tests/conf/remote.conf b/dev/tests/conf/remote.conf index c8d7454..4dbcaf7 100644 --- a/dev/tests/conf/remote.conf +++ b/dev/tests/conf/remote.conf @@ -15,7 +15,7 @@ INITIATOR_SYNC_DIR="${HOME}/osync-tests/initiator" ## Target is the system osync synchronizes to (can be the same system as the initiator in case of local sync tasks). The target directory can be a local or remote path. #TARGET_SYNC_DIR="${HOME}/osync-tests/target" -TARGET_SYNC_DIR="ssh://root@localhost:22/${HOME}/osync-tests/target" +TARGET_SYNC_DIR="ssh://root@localhost:49999/${HOME}/osync-tests/target" ## If the target system is remote, you can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information. SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local" @@ -70,12 +70,12 @@ SSH_COMPRESSION=yes SSH_IGNORE_KNOWN_HOSTS=no ## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync. -REMOTE_HOST_PING=no +REMOTE_HOST_PING=yes ## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync. ## If you use this function, you should set more than one 3rd party host, and be sure you can ping them. ## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute. -REMOTE_3RD_PARTY_HOSTS=www.kernel.org www.google.com +REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com" ## ---------- MISC OPTIONS From 4838d8bd25aff0c8c221dfdda7aea456f6ab3ca7 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 3 Feb 2017 09:54:10 +0100 Subject: [PATCH 16/16] Back to a mergeable state --- dev/tests/conf/remote.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tests/conf/remote.conf b/dev/tests/conf/remote.conf index 4dbcaf7..16ae50e 100644 --- a/dev/tests/conf/remote.conf +++ b/dev/tests/conf/remote.conf @@ -15,7 +15,7 @@ INITIATOR_SYNC_DIR="${HOME}/osync-tests/initiator" ## Target is the system osync synchronizes to (can be the same system as the initiator in case of local sync tasks). The target directory can be a local or remote path. #TARGET_SYNC_DIR="${HOME}/osync-tests/target" -TARGET_SYNC_DIR="ssh://root@localhost:49999/${HOME}/osync-tests/target" +TARGET_SYNC_DIR="ssh://root@localhost:22/${HOME}/osync-tests/target" ## If the target system is remote, you can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information. SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local" @@ -70,7 +70,7 @@ SSH_COMPRESSION=yes SSH_IGNORE_KNOWN_HOSTS=no ## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync. -REMOTE_HOST_PING=yes +REMOTE_HOST_PING=no ## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync. ## If you use this function, you should set more than one 3rd party host, and be sure you can ping them.