diff --git a/dev/common_install.sh b/dev/common_install.sh index 3d8dd77..d5d024e 100755 --- a/dev/common_install.sh +++ b/dev/common_install.sh @@ -6,7 +6,7 @@ PROGRAM=[prgname] PROGRAM_VERSION=[version] PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" -SCRIPT_BUILD=2016121301 +SCRIPT_BUILD=2016122701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -48,6 +48,8 @@ include #### GetLocalOS SUBSET #### function SetLocalOSSettings { USER=root + # LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS + case $LOCAL_OS in *"BSD"*) GROUP=wheel @@ -74,7 +76,7 @@ function SetLocalOSSettings { exit 1 fi - OS=$(UrlEncode "$localOsVar") + OS=$(UrlEncode "$LOCAL_OS_FULL") } function GetInit { diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index f60b139..0b6858a 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2016 by Orsiris de Jong" +AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-RC1 -PROGRAM_BUILD=2016121302 -IS_STABLE=yes +PROGRAM_VERSION=1.2-RC1+dev +PROGRAM_BUILD=2016121901 +IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -40,12 +40,12 @@ IS_STABLE=yes # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -#### OFUNCTIONS MINI SUBSET #### -_OFUNCTIONS_VERSION=2.1-RC1 -_OFUNCTIONS_BUILD=2016121304 +_OFUNCTIONS_VERSION=2.1-RC1+dev +_OFUNCTIONS_BUILD=2017010401 _OFUNCTIONS_BOOTSTRAP=true -## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name @@ -148,6 +148,13 @@ function Dummy { } #### Logger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -178,7 +185,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -186,7 +193,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -227,8 +234,8 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -236,7 +243,7 @@ function RemoteLogger { # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr -# _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels @@ -261,7 +268,7 @@ function Logger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "$prefix($level):$value" "$prefix\e[41m$value\e[0m" true + _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true # ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag. echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" @@ -287,7 +294,7 @@ function Logger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "$prefix$value" "$prefix$value" + _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -300,8 +307,8 @@ function Logger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### @@ -610,7 +617,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } @@ -647,11 +654,6 @@ function Spinner { fi } -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -708,7 +710,7 @@ function WaitForTaskCompletion { Spinner fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -864,7 +866,7 @@ function ParallelExec { fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -902,7 +904,7 @@ function ParallelExec { fi eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) + return $((commandCount - counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -925,7 +927,6 @@ function ParallelExec { if [ $(IsInteger $pid) -eq 1 ]; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) if kill -0 $pid > /dev/null 2>&1; then - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -949,7 +950,7 @@ function ParallelExec { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done return $errorCount @@ -965,11 +966,6 @@ function CleanUp { fi } -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1117,7 +1113,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1167,9 +1163,11 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } -#### OFUNCTIONS MINI SUBSET END #### function GetRemoteOS { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1193,7 +1191,7 @@ function GetOs { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1438,8 +1436,8 @@ function __CheckArguments { if [ "$argument" == "" ]; then fetchArguments=false else - argList="$argList[Argument $(($iterate-2)): $argument] " - iterate=$(($iterate+1)) + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) fi done @@ -1455,7 +1453,7 @@ function __CheckArguments { Logger "Entering function [$callerName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" Logger "$callerName arguments: $argList" "ERROR" else if [ ! -z "$argList" ]; then @@ -1576,7 +1574,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" @@ -1612,6 +1610,50 @@ function PostInit { fi } +function SetCompression { + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + function InitLocalOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1646,8 +1688,12 @@ function InitLocalOSDependingSettings { STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi + + # Set compression first time when we know what local os we have + SetCompression } +# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1740,47 +1786,8 @@ function InitRemoteOSDependingSettings { RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + # Set compression options again after we know what remote OS we're dealing with + SetCompression } ## IFS debug function @@ -1961,24 +1968,26 @@ function CheckCurrentConfigAll { function _CheckReplicasLocal { local replicaPath="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local diskSpace if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + Logger "Local replica path [$replicaPath] does not exist / is not writable." "CRITICAL" return 1 fi fi @@ -2007,7 +2016,9 @@ function _CheckReplicasLocal { function _CheckReplicasRemote { local replicaPath="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -2017,7 +2028,7 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -2041,7 +2052,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } function IsInteger { @@ -2083,6 +2094,13 @@ function HumanToNumeric { echo $value } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -2113,7 +2131,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -2121,7 +2139,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2162,15 +2180,15 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } -function _CheckReplicasRemoteSub { +function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -2179,7 +2197,7 @@ function _CheckReplicasRemoteSub { RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" fi else - RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + RemoteLogger "Remote replica path [$replicaPath] does not exist / is not writable." "CRITICAL" exit 1 fi fi @@ -2212,10 +2230,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Failed to check remote replica." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2238,13 +2256,13 @@ function CheckReplicas { fi fi - _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" & + _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" & pids="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckReplicasLocal "${TARGET[$__replicaDir]}" & + _CheckReplicasLocal "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _CheckReplicasRemote "${TARGET[$__replicaDir]}" & + _CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" fi WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false @@ -2270,11 +2288,11 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 fi fi @@ -2312,11 +2330,11 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval - Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" @@ -2345,7 +2363,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -2369,7 +2387,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } ## Modified version of http://stackoverflow.com/a/8574392 @@ -2399,6 +2417,13 @@ function IsInteger { echo 0 fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -2429,7 +2454,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -2437,7 +2462,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2478,14 +2503,14 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -2537,7 +2562,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -2555,10 +2580,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Remote lock handling failed." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2613,24 +2638,28 @@ function HandleLocks { function _UnlockReplicasLocal { local lockfile="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock local replica." "ERROR" $retval + Logger "Could not unlock local $replicaType replica." "ERROR" $retval else - Logger "Removed local replica lock." "DEBUG" + Logger "Removed local $replicaType replica lock." "DEBUG" fi fi } function _UnlockReplicasRemote { local lockfile="${1}" - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG + local replicaType="${2}" + + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -2640,17 +2669,17 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock remote replica." "ERROR" $retval - Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Could not unlock $replicaType remote replica." "ERROR" $retval + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else - Logger "Removed remote replica lock." "DEBUG" + Logger "Removed remote $replicaType replica lock." "DEBUG" fi } @@ -2664,16 +2693,16 @@ function UnlockReplicas { fi if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then - _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" & + _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" & pids="$!" fi if [ $TARGET_LOCK_FILE_EXISTS == true ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then - _UnlockReplicasLocal "${TARGET[$__lockFile]}" & + _UnlockReplicasLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _UnlockReplicasRemote "${TARGET[$__lockFile]}" & + _UnlockReplicasRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" fi fi @@ -2733,10 +2762,11 @@ function treeList { return $? elif [ $retval -eq 23 ]; then Logger "Some files could not be listed in $replicaType replica [$replicaPath]. Check for failing symlinks." "ERROR" $retval - Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return 0 else Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return $retval fi } @@ -2744,6 +2774,7 @@ function treeList { # deleteList(replicaType): Creates a list of files vanished from last run on replica $1 (initiator/target) function deleteList { local replicaType="${1}" # replica type: initiator, target + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -2801,6 +2832,7 @@ function _getFileCtimeMtimeLocal { local replicaPath="${1}" # Contains replica path local replicaType="${2}" # Initiator / Target local fileList="${3}" # Contains list of files to get time attrs + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -2811,7 +2843,7 @@ function _getFileCtimeMtimeLocal { if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi @@ -2834,14 +2866,14 @@ function _getFileCtimeMtimeRemote { if [ $retval -ne 0 ]; then Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -2851,7 +2883,7 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return $retval else @@ -3080,7 +3112,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -3183,9 +3215,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -3209,9 +3241,16 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -3242,7 +3281,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -3250,7 +3289,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3291,8 +3330,8 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -3304,7 +3343,7 @@ function RemoteLogger { previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -3319,7 +3358,7 @@ function RemoteLogger { if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -3327,11 +3366,11 @@ function RemoteLogger { parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -3347,7 +3386,7 @@ function RemoteLogger { if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -3797,25 +3836,25 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "Directory cleanup complete on $replicaType replica." "NOTICE" fi - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" - elif [ -d "$replicaDeletionPath" ] && ! [ -w "$replicaDeletionPath" ]; then Logger "The $replicaType replica dir [$replicaDeletionPath] is not writable. Cannot clean old files." "ERROR" @@ -3850,13 +3889,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" @@ -3866,7 +3905,7 @@ ENDSSH retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else Logger "Cleanup complete on $replicaType replica." "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" diff --git a/dev/merge.sh b/dev/merge.sh index 93138b2..c2476ed 100755 --- a/dev/merge.sh +++ b/dev/merge.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -## MERGE 2016121101 +## MERGE 2016121901 ## Merges ofunctions.sh and n_program.sh into program.sh ## Adds installer @@ -40,7 +40,7 @@ function __PREPROCESSOR_Constants { __PREPROCESSOR_SUBSETS=( '#### OFUNCTIONS FULL SUBSET ####' - '#### OFUNCTIONS MINIMAL SUBSET ####' + '#### OFUNCTIONS MINI SUBSET ####' '#### _OFUNCTIONS_BOOTSTRAP SUBSET ####' '#### DEBUG SUBSET ####' '#### TrapError SUBSET ####' diff --git a/dev/n_osync.sh b/dev/n_osync.sh index d54e103..ca13e97 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2016 by Orsiris de Jong" +AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-RC1+dev -PROGRAM_BUILD=2016121502 +PROGRAM_BUILD=2016121901 IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG @@ -210,7 +210,7 @@ function _CheckReplicasLocal { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval @@ -220,7 +220,7 @@ function _CheckReplicasLocal { Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + Logger "Local replica path [$replicaPath] does not exist / is not writable." "CRITICAL" return 1 fi fi @@ -261,16 +261,17 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### IsInteger SUBSET #### include #### HumanToNumeric SUBSET #### include #### RemoteLogger SUBSET #### + function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -279,7 +280,7 @@ function _CheckReplicasRemoteSub { RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" fi else - RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + RemoteLogger "Remote replica path [$replicaPath] does not exist / is not writable." "CRITICAL" exit 1 fi fi @@ -370,7 +371,7 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -412,7 +413,7 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -445,7 +446,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### ArrayContains SUBSET #### @@ -454,7 +455,7 @@ include #### RemoteLogger SUBSET #### function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -506,7 +507,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -589,7 +590,7 @@ function _UnlockReplicasLocal { local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then Logger "Could not unlock local $replicaType replica." "ERROR" $retval @@ -613,9 +614,9 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? @@ -817,7 +818,7 @@ function _getFileCtimeMtimeRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -1056,7 +1057,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -1159,9 +1160,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### RemoteLogger SUBSET #### @@ -1174,7 +1175,7 @@ include #### RemoteLogger SUBSET #### previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -1189,7 +1190,7 @@ include #### RemoteLogger SUBSET #### if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -1197,11 +1198,11 @@ include #### RemoteLogger SUBSET #### parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -1217,7 +1218,7 @@ include #### RemoteLogger SUBSET #### if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -1667,7 +1668,7 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval @@ -1676,7 +1677,7 @@ function _SoftDeleteLocal { Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval @@ -1720,13 +1721,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 4f4d8be..f2b8dbd 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,11 +3,12 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016121501 +_OFUNCTIONS_BUILD=2017010401 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### -## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name @@ -113,6 +114,13 @@ function Dummy { #### Logger SUBSET #### #### RemoteLogger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -143,7 +151,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -192,8 +200,8 @@ function RemoteLogger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } #### RemoteLogger SUBSET END #### @@ -202,7 +210,7 @@ function RemoteLogger { # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr -# _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels @@ -253,7 +261,7 @@ function Logger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "$prefix$value" "$prefix$value" + _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -266,8 +274,8 @@ function Logger { return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### @@ -579,7 +587,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } #### TrapError SUBSET END #### @@ -617,11 +625,6 @@ function Spinner { fi } -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -678,7 +681,7 @@ function WaitForTaskCompletion { Spinner fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -834,7 +837,7 @@ function ParallelExec { fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -872,7 +875,7 @@ function ParallelExec { fi eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) + return $((commandCount - counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -895,7 +898,6 @@ function ParallelExec { if [ $(IsInteger $pid) -eq 1 ]; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) if kill -0 $pid > /dev/null 2>&1; then - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -919,7 +921,7 @@ function ParallelExec { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done return $errorCount @@ -935,11 +937,6 @@ function CleanUp { fi } -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1096,7 +1093,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1146,6 +1143,9 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } #### GetLocalOS SUBSET END #### @@ -1173,7 +1173,7 @@ function GetOs { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1418,8 +1418,8 @@ function __CheckArguments { if [ "$argument" == "" ]; then fetchArguments=false else - argList="$argList[Argument $(($iterate-2)): $argument] " - iterate=$(($iterate+1)) + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) fi done @@ -1435,7 +1435,7 @@ function __CheckArguments { Logger "Entering function [$callerName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" Logger "$callerName arguments: $argList" "ERROR" else if [ ! -z "$argList" ]; then @@ -1556,7 +1556,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" @@ -1592,6 +1592,50 @@ function PostInit { fi } +function SetCompression { + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + function InitLocalOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1626,8 +1670,12 @@ function InitLocalOSDependingSettings { STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi + + # Set compression first time when we know what local os we have + SetCompression } +# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1720,47 +1768,8 @@ function InitRemoteOSDependingSettings { RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + # Set compression options again after we know what remote OS we're dealing with + SetCompression } ## IFS debug function diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 42e6fff..68143fc 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -7,7 +7,7 @@ ## On CYGWIN / MSYS, ACL and extended attributes aren't supported -# osync test suite 2016121301 +# osync test suite 2016122302 # 4 tests: # quicklocal @@ -199,11 +199,21 @@ function oneTimeSetUp () { if [ "$TRAVIS_RUN" == true ]; then echo "Running with travis settings" REMOTE_USER="travis" + RHOST_PING="no" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" + SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" "no" + + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOSTS" "" + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "no" else echo "Running with local settings" REMOTE_USER="root" + RHOST_PING="yes" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" + SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" "yes" + + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" + SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "yes" fi # Get default ssh port from env @@ -311,7 +321,7 @@ function test_LargeFileSet () { PrepareLocalDirs DownloadLargeFileSet "$INITIATOR_DIR" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "LargeFileSet test with parameters [$i]." "0" $? [ -d "$INITIATOR_DIR/$OSYNC_STATE_DIR" ] @@ -339,7 +349,7 @@ function test_Exclusions () { numberOfPHPFiles=$(find "$INITIATOR_DIR" ! -wholename "$INITIATOR_DIR/$OSYNC_WORKDIR*" -name "*.php" | wc -l) - REMOTE_HOST_PING=no RSYNC_EXCLUDE_PATTERN="*.php" ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING RSYNC_EXCLUDE_PATTERN="*.php" ./$OSYNC_EXECUTABLE $i assertEquals "Exclusions with parameters [$i]." "0" $? numberOfInitiatorFiles=$(find "$INITIATOR_DIR" ! -wholename "$INITIATOR_DIR/$OSYNC_WORKDIR*" | wc -l) @@ -366,13 +376,13 @@ function test_Deletetion () { touch "$tFile1" touch "$tFile2" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? rm -f "$iFile1" rm -f "$tFile1" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second deletion run with parameters [$i]." "0" $? [ -f "$TARGET_DIR/$OSYNC_DELETE_DIR/$(basename $iFile1)" ] @@ -417,7 +427,7 @@ function test_deletion_failure () { touch "$INITIATOR_DIR/$FileA" touch "$TARGET_DIR/$FileB" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? rm -f "$INITIATOR_DIR/$FileA" @@ -428,7 +438,7 @@ function test_deletion_failure () { $SUDO_CMD $IMMUTABLE_ON_CMD "$INITIATOR_DIR/$FileB" # This shuold fail with exitcode 1 - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second deletion run with parameters [$i]." "1" $? # standard file tests @@ -446,7 +456,7 @@ function test_deletion_failure () { $SUDO_CMD $IMMUTABLE_OFF_CMD "$TARGET_DIR/$FileA" $SUDO_CMD $IMMUTABLE_OFF_CMD "$INITIATOR_DIR/$FileB" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Third deletion run with parameters [$i]." "0" $? [ ! -f "$TARGET_DIR/$FileA" ] @@ -498,14 +508,14 @@ function test_skip_deletion () { touch "$TARGET_DIR/$FileB" # First run - REMOTE_HOST_PING=no SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? rm -f "$INITIATOR_DIR/$FileA" rm -f "$TARGET_DIR/$FileB" # Second run - REMOTE_HOST_PING=no SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING SKIP_DELETION="$mode" ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? if [ "$mode" == "initiator" ]; then @@ -578,14 +588,14 @@ function test_handle_symlinks () { ln -s "$INITIATOR_DIR/$FileA" "$INITIATOR_DIR/$FileAL" ln -s "$TARGET_DIR/$FileB" "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First symlink run with parameters [$i]." "0" $? # Delete symlinks rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second symlink deletion run with parameters [$i]." "0" $? # symlink deletion propagation @@ -604,7 +614,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileA" rm -f "$TARGET_DIR/$FileB" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Third broken symlink run with parameters [$i]." "0" $? [ -L "$TARGET_DIR/$FileAL" ] @@ -617,7 +627,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Fourth symlink deletion run with parameters [$i]." "0" $? [ ! -L "$TARGET_DIR/$FileAL" ] @@ -665,14 +675,14 @@ function test_handle_symlinks () { ln -s "$INITIATOR_DIR/$FileA" "$INITIATOR_DIR/$FileAL" ln -s "$TARGET_DIR/$FileB" "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First symlink run with parameters [$i]." "0" $? # Delete symlinks rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Second symlink deletion run with parameters [$i]." "0" $? # symlink deletion propagation @@ -691,7 +701,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileA" rm -f "$TARGET_DIR/$FileB" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Third broken symlink run with parameters should fail [$i]." "1" $? [ ! -f "$TARGET_DIR/$FileAL" ] @@ -704,7 +714,7 @@ function test_handle_symlinks () { rm -f "$INITIATOR_DIR/$FileAL" rm -f "$TARGET_DIR/$FileBL" - COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + COPY_SYMLINKS=$copySymlinks REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Fourth symlink deletion run should resume with parameters [$i]." "0" $? [ ! -f "$TARGET_DIR/$FileAL" ] @@ -735,7 +745,7 @@ function test_softdeletion_cleanup () { PrepareLocalDirs # First run - #REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + #REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i #assertEquals "First deletion run with parameters [$i]." "0" $? # Get current drive @@ -764,7 +774,7 @@ function test_softdeletion_cleanup () { fi # Second run - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i # Check file presence for file in "${files[@]}"; do @@ -828,7 +838,7 @@ function test_FileAttributePropagation () { touch "$TARGET_DIR/$FileB" # First run - PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? sleep 1 @@ -856,7 +866,7 @@ function test_FileAttributePropagation () { assertEquals "Set ACL on target directory" "0" $? # Second run - PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + PRESERVE_ACL=yes PRESERVE_XATTR=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? getfacl "$TARGET_DIR/$FileA" | grep "other::r-x" > /dev/null @@ -891,14 +901,14 @@ function test_ConflictBackups () { echo "$FileB" > "$TARGET_DIR/$FileB" # First run - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA+" > "$TARGET_DIR/$FileA" echo "$FileB+" > "$INITIATOR_DIR/$FileB" # Second run - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "First deletion run with parameters [$i]." "0" $? [ -f "$INITIATOR_DIR/$OSYNC_BACKUP_DIR/$FileA" ] @@ -935,28 +945,28 @@ function test_MultipleConflictBackups () { echo "$FileB" > "$TARGET_DIR/$FileB" # First run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA+" > "$TARGET_DIR/$FileA" echo "$FileB+" > "$INITIATOR_DIR/$FileB" # Second run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA-" > "$TARGET_DIR/$FileA" echo "$FileB-" > "$INITIATOR_DIR/$FileB" # Third run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? echo "$FileA*" > "$TARGET_DIR/$FileA" echo "$FileB*" > "$INITIATOR_DIR/$FileB" # Fouth run - CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i $additionalParameters + CONFLICT_BACKUP_MULTIPLE=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i $additionalParameters assertEquals "First deletion run with parameters [$i]." "0" $? # This test may fail only on 31th December at 23:59 :) @@ -986,12 +996,12 @@ function test_Locking () { mkdir -p "$INITIATOR_DIR/$OSYNC_WORKDIR" echo 65536 > "$INITIATOR_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Should be able to resume when initiator has lock without running pid." "0" $? echo $$ > "$INITIATOR_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Should never be able to resume when initiator has lock with running pid." "1" $? done @@ -1000,14 +1010,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@quicklocal > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} assertEquals "Should be able to resume locked target with same instance_id in quickLocal mode." "0" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@local > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} assertEquals "Should be able to resume locked target with same instance_id in confLocal mode." "0" $? if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ]; then @@ -1015,14 +1025,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@quickremote > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} assertEquals "Should be able to resume locked target with same instance_id in quickRemote mode." "0" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@remote > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} assertEquals "Should be able to resume locked target with same instance_id in confRemote mode." "0" $? fi @@ -1031,14 +1041,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickLocal]} assertEquals "Should be able to resume locked local target with bogus instance id in quickLocal mode." "0" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} assertEquals "Should be able to resume locked local target with bogus instance_id in confLocal mode." "0" $? if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ]; then @@ -1046,14 +1056,14 @@ function test_Locking () { mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__quickRemote]} assertEquals "Should not be able to resume remote locked target with bogus instance_id in quickRemote mode." "1" $? PrepareLocalDirs mkdir -p "$TARGET_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$TARGET_DIR/$OSYNC_WORKDIR/lock" - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confRemote]} assertEquals "Should not be able to resume remote locked target with bogus instance_id in confRemote mode." "1" $? fi @@ -1070,7 +1080,7 @@ function test_Locking () { mkdir -p "$INITIATOR_DIR/$OSYNC_WORKDIR" echo 65536@bogusinstance > "$INITIATOR_DIR/$OSYNC_WORKDIR/lock" - FORCE_STRANGER_UNLOCK=yes REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE $i + FORCE_STRANGER_UNLOCK=yes REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE $i assertEquals "Should be able to resume when target has lock with different instance id but FORCE_STRANGER_UNLOCK=yes." "0" $? done @@ -1295,6 +1305,10 @@ function test_UpgradeConfRun () { ./$OSYNC_UPGRADE "$CONF_DIR/$TMP_OLD_CONF" assertEquals "Conf file upgrade" "0" $? + + # Update remote conf files with SSH port + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${HOME}/osync-tests/target#' "$CONF_DIR/$TMP_OLD_CONF" + ./$OSYNC_EXECUTABLE "$CONF_DIR/$TMP_OLD_CONF" assertEquals "Upgraded conf file execution test" "0" $? @@ -1363,7 +1377,7 @@ function test_NoRemoteAccessTest () { cd "$OSYNC_DIR" PrepareLocalDirs - REMOTE_HOST_PING=no ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} + REMOTE_HOST_PING=$RHOST_PING ./$OSYNC_EXECUTABLE ${osyncParameters[$__confLocal]} assertEquals "Basic local test without remote access." "0" $? } diff --git a/install.sh b/install.sh index b30dc11..c67c1e3 100755 --- a/install.sh +++ b/install.sh @@ -3,10 +3,10 @@ _OFUNCTIONS_BOOTSTRAP=true PROGRAM=osync -PROGRAM_VERSION=1.2-RC1 +PROGRAM_VERSION=1.2-RC1+dev PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" -SCRIPT_BUILD=2016121301 +SCRIPT_BUILD=2016122701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -92,7 +92,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -142,10 +142,15 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } function SetLocalOSSettings { USER=root + # LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS + case $LOCAL_OS in *"BSD"*) GROUP=wheel @@ -172,7 +177,7 @@ function SetLocalOSSettings { exit 1 fi - OS=$(UrlEncode "$localOsVar") + OS=$(UrlEncode "$LOCAL_OS_FULL") } function GetInit { diff --git a/osync.sh b/osync.sh index 10776a5..a23cfd3 100755 --- a/osync.sh +++ b/osync.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2016 by Orsiris de Jong" +AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-RC1 -PROGRAM_BUILD=2016121302 -IS_STABLE=yes +PROGRAM_VERSION=1.2-RC1+dev +PROGRAM_BUILD=2016121901 +IS_STABLE=no -#### OFUNCTIONS MINI SUBSET #### -_OFUNCTIONS_VERSION=2.1-RC1 -_OFUNCTIONS_BUILD=2016121304 +_OFUNCTIONS_VERSION=2.1-RC1+dev +_OFUNCTIONS_BUILD=2017010401 _OFUNCTIONS_BOOTSTRAP=true -## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name @@ -112,6 +112,13 @@ function Dummy { } #### Logger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -142,7 +149,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -150,7 +157,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -186,8 +193,8 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -195,7 +202,7 @@ function RemoteLogger { # Environment variables # _LOGGER_SILENT: Disables any output to stdout & stderr -# _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel # _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout # Loglevels @@ -220,7 +227,7 @@ function Logger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "$prefix($level):$value" "$prefix\e[41m$value\e[0m" true + _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true # ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag. echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" @@ -246,7 +253,7 @@ function Logger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "$prefix$value" "$prefix$value" + _Logger "$prefix$value" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -254,8 +261,8 @@ function Logger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } #### Logger SUBSET END #### @@ -560,7 +567,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } @@ -596,11 +603,6 @@ function Spinner { fi } -# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array -# usage: joinString separaratorChar Array -function joinString { - local IFS="$1"; shift; echo "$*"; -} # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -654,7 +656,7 @@ function WaitForTaskCompletion { Spinner fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -803,7 +805,7 @@ function ParallelExec { fi if [ $counting == true ]; then - exec_time=$(($SECONDS - $seconds_begin)) + exec_time=$((SECONDS - seconds_begin)) else exec_time=$SECONDS fi @@ -841,7 +843,7 @@ function ParallelExec { fi eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) + return $((commandCount - counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -864,7 +866,6 @@ function ParallelExec { if [ $(IsInteger $pid) -eq 1 ]; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) if kill -0 $pid > /dev/null 2>&1; then - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -884,7 +885,7 @@ function ParallelExec { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done return $errorCount @@ -899,11 +900,6 @@ function CleanUp { fi } -# obsolete, use StripQuotes -function SedStripQuotes { - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1051,7 +1047,7 @@ function GetLocalOS { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1101,9 +1097,11 @@ function GetLocalOS { if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$localOsVar]." "DEBUG" fi + + # Add a global variable for statistics in installer + LOCAL_OS_FULL="$localOsVar" } -#### OFUNCTIONS MINI SUBSET END #### function GetRemoteOS { @@ -1126,7 +1124,7 @@ function GetOs { if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then localOsVar="Microsoft" else - localOsVar="$(uname -spio 2>&1)" + localOsVar="$(uname -spior 2>&1)" if [ $? != 0 ]; then localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then @@ -1440,7 +1438,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" @@ -1475,6 +1473,50 @@ function PostInit { fi } +function SetCompression { + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + function InitLocalOSDependingSettings { ## If running under Msys, some commands do not run the same way @@ -1508,8 +1550,12 @@ function InitLocalOSDependingSettings { STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi + + # Set compression first time when we know what local os we have + SetCompression } +# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then @@ -1601,47 +1647,8 @@ function InitRemoteOSDependingSettings { RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + # Set compression options again after we know what remote OS we're dealing with + SetCompression } ## IFS debug function @@ -1819,23 +1826,25 @@ function CheckCurrentConfigAll { function _CheckReplicasLocal { local replicaPath="${1}" + local replicaType="${2}" + local retval local diskSpace if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + Logger "Local replica path [$replicaPath] does not exist / is not writable." "CRITICAL" return 1 fi fi @@ -1864,6 +1873,8 @@ function _CheckReplicasLocal { function _CheckReplicasRemote { local replicaPath="${1}" + local replicaType="${2}" + local retval local cmd @@ -1873,7 +1884,7 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then @@ -1893,7 +1904,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } function IsInteger { @@ -1935,6 +1946,13 @@ function HumanToNumeric { echo $value } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -1965,7 +1983,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -1973,7 +1991,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2009,15 +2027,15 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } -function _CheckReplicasRemoteSub { +function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -2026,7 +2044,7 @@ function _CheckReplicasRemoteSub { RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" fi else - RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + RemoteLogger "Remote replica path [$replicaPath] does not exist / is not writable." "CRITICAL" exit 1 fi fi @@ -2059,10 +2077,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Failed to check remote replica." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2084,13 +2102,13 @@ function CheckReplicas { fi fi - _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" & + _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" & pids="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckReplicasLocal "${TARGET[$__replicaDir]}" & + _CheckReplicasLocal "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _CheckReplicasRemote "${TARGET[$__replicaDir]}" & + _CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & pids="$pids;$!" fi WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false @@ -2115,11 +2133,11 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 fi fi @@ -2157,11 +2175,11 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval - Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" return 1 else Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" @@ -2189,7 +2207,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then @@ -2209,7 +2227,7 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } ## Modified version of http://stackoverflow.com/a/8574392 @@ -2239,6 +2257,13 @@ function IsInteger { echo 0 fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -2269,7 +2294,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -2277,7 +2302,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2313,14 +2338,14 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -2372,7 +2397,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -2390,10 +2415,10 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Remote lock handling failed." "CRITICAL" $retval fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then ( _LOGGER_PREFIX="" - Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" ) fi if [ $retval -ne 0 ]; then @@ -2447,22 +2472,26 @@ function HandleLocks { function _UnlockReplicasLocal { local lockfile="${1}" + local replicaType="${2}" + local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock local replica." "ERROR" $retval + Logger "Could not unlock local $replicaType replica." "ERROR" $retval else - Logger "Removed local replica lock." "DEBUG" + Logger "Removed local $replicaType replica lock." "DEBUG" fi fi } function _UnlockReplicasRemote { local lockfile="${1}" + local replicaType="${2}" + local retval local cmd @@ -2472,17 +2501,17 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? if [ $retval -ne 0 ]; then - Logger "Could not unlock remote replica." "ERROR" $retval - Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Could not unlock $replicaType remote replica." "ERROR" $retval + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else - Logger "Removed remote replica lock." "DEBUG" + Logger "Removed remote $replicaType replica lock." "DEBUG" fi } @@ -2495,16 +2524,16 @@ function UnlockReplicas { fi if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then - _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" & + _UnlockReplicasLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" & pids="$!" fi if [ $TARGET_LOCK_FILE_EXISTS == true ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then - _UnlockReplicasLocal "${TARGET[$__lockFile]}" & + _UnlockReplicasLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" else - _UnlockReplicasRemote "${TARGET[$__lockFile]}" & + _UnlockReplicasRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & pids="$pids;$!" fi fi @@ -2563,10 +2592,11 @@ function treeList { return $? elif [ $retval -eq 23 ]; then Logger "Some files could not be listed in $replicaType replica [$replicaPath]. Check for failing symlinks." "ERROR" $retval - Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return 0 else Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval + Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return $retval fi } @@ -2575,6 +2605,7 @@ function treeList { function deleteList { local replicaType="${1}" # replica type: initiator, target + local retval local subretval local cmd @@ -2631,6 +2662,7 @@ function _getFileCtimeMtimeLocal { local replicaType="${2}" # Initiator / Target local fileList="${3}" # Contains list of files to get time attrs + local retval echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -2639,7 +2671,7 @@ function _getFileCtimeMtimeLocal { if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi @@ -2661,14 +2693,14 @@ function _getFileCtimeMtimeRemote { if [ $retval -ne 0 ]; then Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return 1 fi $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -2678,7 +2710,7 @@ ENDSSH if [ $retval -ne 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi return $retval else @@ -2904,7 +2936,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -3006,9 +3038,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then @@ -3028,9 +3060,16 @@ function TrapError { local code="${2:-1}" if [ $_LOGGER_SILENT == false ]; then - echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + # Sub function of Logger function _Logger { local logValue="${1}" # Log to file @@ -3061,7 +3100,7 @@ function RemoteLogger { local retval="${3:-undef}" # optional return value of command if [ "$_LOGGER_PREFIX" == "time" ]; then - prefix="RTIME: $SECONDS - " + prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="R $(date) - " else @@ -3069,7 +3108,7 @@ function RemoteLogger { fi if [ "$level" == "CRITICAL" ]; then - _Logger "" "$prefix\e[41m$value\e[0m" true + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3105,8 +3144,8 @@ function RemoteLogger { return fi else - _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _Logger "Value was: $prefix$value" + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } @@ -3118,7 +3157,7 @@ function RemoteLogger { previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -3133,7 +3172,7 @@ function RemoteLogger { if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -3141,11 +3180,11 @@ function RemoteLogger { parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -3161,7 +3200,7 @@ function RemoteLogger { if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -3608,25 +3647,25 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "Directory cleanup complete on $replicaType replica." "NOTICE" fi - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" - elif [ -d "$replicaDeletionPath" ] && ! [ -w "$replicaDeletionPath" ]; then Logger "The $replicaType replica dir [$replicaDeletionPath] is not writable. Cannot clean old files." "ERROR" @@ -3660,13 +3699,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" @@ -3676,7 +3715,7 @@ ENDSSH retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" $retval - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" else Logger "Cleanup complete on $replicaType replica." "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"