diff --git a/dev/bootstrap.sh b/dev/bootstrap.sh new file mode 100755 index 0000000..d595e6a --- /dev/null +++ b/dev/bootstrap.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +## dev pre-processor bootstrap rev 2016120701 +## Yeah !!! A really tech sounding name... In fact it's just include emulation in bash + +outputFileName="$0" + +source "merge.sh" +__PREPROCESSOR_PROGRAM=osync +__PREPROCESSOR_Constants + +cp "n_$__PREPROCESSOR_PROGRAM.sh" "$outputFileName.tmp.sh" +if [ $? != 0 ]; then + QuickLogger "Cannot copy original file [n_$__PREPROCESSOR_PROGRAM.sh] to [$outputFileName.tmp.sh]." "stderr" + exit 1 +fi +for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do + __PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "$outputFileName.tmp.sh" +done +chmod +x "$0.tmp.sh" +if [ $? != 0 ]; then + QuickLogger "Cannot make [$outputFileName] executable.." "stderr" + exit 1 +fi + +"$outputFileName.tmp.sh" "$@" + diff --git a/dev/merge.sh b/dev/merge.sh index 994e635..2fd6625 100755 --- a/dev/merge.sh +++ b/dev/merge.sh @@ -1,142 +1,156 @@ #!/usr/bin/env bash -## MERGE 2016112001 +## MERGE 2016120802 ## Merges ofunctions.sh and n_program.sh into program.sh ## Adds installer -PROGRAM=osync -VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh) -VERSION=${VERSION#*=} +function __PREPROCESSOR_Merge { + PROGRAM=osync + VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh) + VERSION=${VERSION#*=} -PARANOIA_DEBUG_LINE="__WITH_PARANOIA_DEBUG" -PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG" -PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG" -MINIMUM_FUNCTION_BEGIN="#### MINIMAL-FUNCTION-SET BEGIN ####" -MINIMUM_FUNCTION_END="#### MINIMAL-FUNCTION-SET END ####" + __PREPROCESSOR_Constants -source "ofunctions.sh" -if [ $? != 0 ]; then - echo "Please run $0 in dev directory with ofunctions.sh" - exit 1 -fi - -function Unexpand { - unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh -} - -function MergeAll { - - sed "/source \"\.\/ofunctions.sh\"/r ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh + source "ofunctions.sh" if [ $? != 0 ]; then - QuickLogger "Cannot sed ofunctions" "stdout" + echo "Please run $0 in dev directory with ofunctions.sh" exit 1 fi - chmod +x debug_$PROGRAM.sh + + __PREPROCESSOR_Unexpand "n_$PROGRAM.sh" "debug_$PROGRAM.sh" + + for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do + __PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "debug_$PROGRAM.sh" + done + + __PREPROCESSOR_CleanDebug + __PREPROCESSOR_CopyCommons + rm -f tmp_$PROGRAM.sh if [ $? != 0 ]; then - QuickLogger "Cannot chmod $PROGRAM.sh" "stdout" + QuickLogger "Cannot remove tmp_$PROGRAM.sh" "stderr" exit 1 fi } -function MergeMinimum { - sed -n "/$MINIMUM_FUNCTION_BEGIN/,/$MINIMUM_FUNCTION_END/p" ofunctions.sh > tmp_minimal.sh - if [ $? != 0 ]; then - QuickLogger "Cannot sed minimum functions." "stdout" - exit 1 - fi - sed "/source \"\.\/ofunctions.sh\"/r tmp_minimal.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' | grep -v "$PARANOIA_DEBUG_LINE" > debug_$PROGRAM.sh - if [ $? != 0 ]; then - QuickLogger "Cannot remove PARANOIA_DEBUG code from tmp_minimum.." "stdout" - exit 1 - fi - rm -f tmp_minimal.sh - if [ $? != 0 ]; then - QuickLogger "Cannot remove tmp_minimal.sh" "stdout" - exit 1 - fi - - chmod +x debug_$PROGRAM.sh - if [ $? != 0 ]; then - QuickLogger "Cannot chmod debug_$PROGRAM.sh" "stdout" - exit 1 - fi +function __PREPROCESSOR_Constants { + PARANOIA_DEBUG_LINE="#__WITH_PARANOIA_DEBUG" + PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG" + PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG" + __PREPROCESSOR_SUBSETS=( + '#### OFUNCTIONS FULL SUBSET ####' + '#### OFUNCTIONS MINIMAL SUBSET ####' + '#### DEBUG SUBSET ####' + '#### TrapError SUBSET ####' + '#### RemoteLogger SUBSET ####' + '#### IsInteger SUBSET ####' + '#### HumanToNumeric SUBSET ####' + '#### ArrayContains SUBSET ####' + ) } +function __PREPROCESSOR_Unexpand { + local source="${1}" + local destination="${2}" -function CleanDebug { + unexpand "$source" > "$destination" + if [ $? != 0 ]; then + QuickLogger "Cannot unexpand [$source] to [$destination]." "stderr" + exit 1 + fi +} -# sed explanation -#/pattern1/{ # if pattern1 is found -# p # print it -# :a # loop -# N # and accumulate lines -# /pattern2/!ba # until pattern2 is found -# s/.*\n// # delete the part before pattern2 -#} -#p -# sed -n '/'$PARANOIA_DEBUG_BEGIN'/{p; :a; N; /'$PARANOIA_DEBUG_END'/!ba; s/.*\n//}; p' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh +function __PREPROCESSOR_MergeSubset { + local subsetBegin="${1}" + local subsetEnd="${2}" + local subsetFile="${3}" + local mergedFile="${4}" - # Way simpler version of the above, compatible with BSD + sed -n "/$subsetBegin/,/$subsetEnd/p" "$subsetFile" > "$subsetFile.$subsetBegin" + if [ $? != 0 ]; then + QuickLogger "Cannot sed subset [$subsetBegin -- $subsetEnd] in [$subsetFile]." "stderr" + exit 1 + fi + sed "/include $subsetBegin/r $subsetFile.$subsetBegin" "$mergedFile" | grep -v -E "$subsetBegin\$|$subsetEnd\$" > "$mergedFile.tmp" + if [ $? != 0 ]; then + QuickLogger "Cannot add subset [$subsetBegin] to [$mergedFile]." "stderr" + exit 1 + fi + rm -f "$subsetFile.$subsetBegin" + if [ $? != 0 ]; then + QuickLogger "Cannot remove temporary subset [$subsetFile.$subsetBeign]." "stderr" + exit 1 + fi + + rm -f "$mergedFile" + if [ $? != 0 ]; then + QuickLogger "Cannot remove merged original file [$mergedFile]." "stderr" + exit 1 + fi + + mv "$mergedFile.tmp" "$mergedFile" + if [ $? != 0 ]; then + QuickLogger "Cannot move merged tmp file to original [$mergedFile]." "stderr" + exit 1 + fi +} + +function __PREPROCESSOR_CleanDebug { sed '/'$PARANOIA_DEBUG_BEGIN'/,/'$PARANOIA_DEBUG_END'/d' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh if [ $? != 0 ]; then - QuickLogger "Cannot remove PARANOIA_DEBUG code from standard build." "stdout" + QuickLogger "Cannot remove PARANOIA_DEBUG code from standard build." "stderr" exit 1 fi - chmod +x ../$PROGRAM.sh + chmod +x "debug_$PROGRAM.sh" if [ $? != 0 ]; then - QuickLogger "Cannot chmod $PROGRAM.sh" "stdout" + QuickLogger "Cannot chmod debug_$PROGRAM.sh" "stderr" + exit 1 + fi + chmod +x "../$PROGRAM.sh" + if [ $? != 0 ]; then + QuickLogger "Cannot chmod $PROGRAM.sh" "stderr" exit 1 fi } -function CopyCommons { +function __PREPROCESSOR_CopyCommons { sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../tmp_install.sh if [ $? != 0 ]; then - QuickLogger "Cannot assemble install." "stdout" + QuickLogger "Cannot assemble install." "stderr" exit 1 fi sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh if [ $? != 0 ]; then - QuickLogger "Cannot change install version." "stdout" + QuickLogger "Cannot change install version." "stderr" exit 1 fi if [ -f "common_batch.sh" ]; then sed "s/\[prgname\]/$PROGRAM/g" common_batch.sh > ../$PROGRAM-batch.sh if [ $? != 0 ]; then - QuickLogger "Cannot assemble batch runner." "stdout" + QuickLogger "Cannot assemble batch runner." "stderr" exit 1 fi chmod +x ../$PROGRAM-batch.sh if [ $? != 0 ]; then - QuickLogger "Cannot chmod $PROGRAM-batch.sh" "stdout" + QuickLogger "Cannot chmod $PROGRAM-batch.sh" "stderr" exit 1 fi fi chmod +x ../install.sh if [ $? != 0 ]; then - QuickLogger "Cannot chmod install.sh" "stdout" + QuickLogger "Cannot chmod install.sh" "stderr" exit 1 fi rm -f ../tmp_install.sh if [ $? != 0 ]; then - QuickLogger "Cannot chmod $PROGRAM.sh" "stdout" + QuickLogger "Cannot chmod $PROGRAM.sh" "stderr" exit 1 fi } -Unexpand -if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then - MergeAll -else - MergeMinimum -fi -CleanDebug -CopyCommons -rm -f tmp_$PROGRAM.sh -if [ $? != 0 ]; then - QuickLogger "Cannot remove tmp_$PROGRAM.sh" "stdout" - exit 1 +# If sourced don't do anything +if [ "$(basename $0)" == "merge.sh" ]; then + __PREPROCESSOR_Merge fi diff --git a/dev/n_osync.sh b/dev/n_osync.sh index a597dd3..75efc3e 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,9 +4,15 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-beta3 -PROGRAM_BUILD=2016120601 +PROGRAM_BUILD=2016120903 IS_STABLE=no +#TODO: update waitfor parallelexec and checkarguments +#TODO: update coding style checkarguments +#TODO: update common install with includes +#TODO: revamp quicklogger +#TODO: is debug subset relevant ? + # Execution order #__WITH_PARANOIA_DEBUG # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -18,13 +24,10 @@ IS_STABLE=no # PostInit no #__WITH_PARANOIA_DEBUG # GetRemoteOS no #__WITH_PARANOIA_DEBUG # InitRemoteOSDependingSettings no #__WITH_PARANOIA_DEBUG -# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG -# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG +# CheckReplicas yes #__WITH_PARANOIA_DEBUG # RunBeforeHook yes #__WITH_PARANOIA_DEBUG # Main no #__WITH_PARANOIA_DEBUG -# CreateStateDirs yes #__WITH_PARANOIA_DEBUG -# CheckLocks yes #__WITH_PARANOIA_DEBUG -# WriteLockFiles yes #__WITH_PARANOIA_DEBUG +# HandleLocks yes #__WITH_PARANOIA_DEBUG # Sync no #__WITH_PARANOIA_DEBUG # treeList yes #__WITH_PARANOIA_DEBUG # treeList yes #__WITH_PARANOIA_DEBUG @@ -43,7 +46,11 @@ IS_STABLE=no # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -source "./ofunctions.sh" +include #### OFUNCTIONS FULL SUBSET #### + +# Hopefully multishell compatible (bash & csh seem to like it) +[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 + _LOGGER_PREFIX="time" ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc @@ -116,7 +123,7 @@ function TrapQuit { } function CheckEnvironment { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" == "yes" ]; then if ! type ssh > /dev/null 2>&1 ; then @@ -134,11 +141,16 @@ function CheckEnvironment { Logger "rsync not present. Sync cannot start." "CRITICAL" exit 1 fi + + if ! type pgrep > /dev/null 2>&1 ; then + Logger "pgrep not present. Sync cannot start." "CRITICAL" + exit 1 + fi } # Only gets checked in config file mode where all values should be present function CheckCurrentConfig { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Check all variables that should contain "yes" or "no" declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) @@ -157,7 +169,7 @@ function CheckCurrentConfig { # Gets checked in quicksync and config file mode function CheckCurrentConfigAll { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local tmp @@ -184,7 +196,7 @@ function CheckCurrentConfigAll { if [ "$SKIP_DELETION" != "" ]; then tmp="$SKIP_DELETION" IFS=',' read -r -a SKIP_DELETION <<< "$tmp" - if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then + if [ $(ArrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ] && [ $(ArrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then Logger "Bogus skip deletion parameter [$SKIP_DELETION]." "CRITICAL" exit 1 fi @@ -193,61 +205,124 @@ function CheckCurrentConfigAll { ###### Osync specific functions (non shared) -function _CheckReplicaPathsLocal { - local replica_path="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG +function _CheckReplicasLocal { + local replicaPath="${1}" + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG - if [ ! -w "$replica_path" ]; then - Logger "Local replica path [$replica_path] is not writable." "CRITICAL" - exit 1 - fi + local diskSpace - if [ ! -d "$replica_path" ]; then + if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replica_path" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 + $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 if [ $? != 0 ]; then - Logger "Cannot create local replica path [$replica_path]." "CRITICAL" + Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" - exit 1 + return 1 else - Logger "Created local replica path [$replica_path]." "NOTICE" + Logger "Created local replica path [$replicaPath]." "NOTICE" fi else - Logger "Local replica path [$replica_path] does not exist." "CRITICAL" - exit 1 + Logger "Local replica path [$replicaPath] does not exist." "CRITICAL" + return 1 + fi + fi + + if [ ! -w "$replicaPath" ]; then + Logger "Local replica path [$replicaPath] is not writable." "CRITICAL" + return 1 + fi + + Logger "Checking minimum disk space in local replica [$replicaPath]." "NOTICE" + diskSpace=$($DF_CMD "$replicaPath" | tail -1 | awk '{print $4}') + if [ $? != 0 ]; then + Logger "Cannot get free space." "ERROR" + else + # Ugly fix for df in some busybox environments that can only show human formats + if [ $(IsInteger $diskSpace) -eq 0 ]; then + diskSpace=$(HumanToNumeric $diskSpace) + fi + + if [ $diskSpace -lt $MINIMUM_SPACE ]; then + Logger "There is not enough free space on local replica [$replicaPath] ($diskSpace KB)." "WARN" fi fi } -function _CheckReplicaPathsRemote { - local replica_path="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG +function _CheckReplicasRemote { + local replicaPath="${1}" + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local cmd CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - if [ $? != 0 ]; then - Logger "Remote replica path [$replica_path] is not writable." "CRITICAL" +$SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 +include #### DEBUG SUBSET #### +include #### TrapError SUBSET #### +include #### IsInteger SUBSET #### +include #### HumanToNumeric SUBSET #### +include #### RemoteLogger SUBSET #### +function _CheckReplicasRemoteSub { + + if [ ! -d "$replicaPath" ]; then + if [ "$CREATE_DIRS" == "yes" ]; then + $COMMAND_SUDO mkdir -p "$replicaPath" + if [ $? != 0 ]; then + RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" + exit 1 + else + RemoteLogger "Created remote replica path [$replicaPath]." "NOTICE" + fi + else + RemoteLogger "Remote replica path [$replicaPath] does not exist." "CRITICAL" + exit 1 + fi + fi + + if [ ! -w "$replicaPath" ]; then + RemoteLogger "Remote replica path [$replicaPath] is not writable." "CRITICAL" exit 1 fi - cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" + RemoteLogger "Checking minimum disk space in remote replica [$replicaPath]." "NOTICE" + diskSpace=$($DF_CMD "$replicaPath" | tail -1 | awk '{print $4}') if [ $? != 0 ]; then - Logger "Cannot create remote replica path [$replica_path]." "CRITICAL" - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" - exit 1 + RemoteLogger "Cannot get free space." "ERROR" + else + # Ugly fix for df in some busybox environments that can only show human formats + if [ $(IsInteger $diskSpace) -eq 0 ]; then + diskSpace=$(HumanToNumeric $diskSpace) + fi + + if [ $diskSpace -lt $MINIMUM_SPACE ]; then + RemoteLogger "There is not enough free space on remote replica [$replicaPath] ($diskSpace KB)." "WARN" + fi + fi +} +_CheckReplicasRemoteSub +exit $? +ENDSSH + result=$? + if [ $result != 0 ]; then + Logger "Failed to check remote replica." "CRITICAL" + fi + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then + ( + _LOGGER_PREFIX="" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" + ) + fi + if [ $result != 0 ]; then + return 1 + else + return 0 fi } -function CheckReplicaPaths { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG +function CheckReplicas { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -258,177 +333,57 @@ function CheckReplicaPaths { fi fi - _CheckReplicaPathsLocal "${INITIATOR[$__replicaDir]}" & + _CheckReplicasLocal "${INITIATOR[$__replicaDir]}" & pids="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckReplicaPathsLocal "${TARGET[$__replicaDir]}" & + _CheckReplicasLocal "${TARGET[$__replicaDir]}" & pids="$pids;$!" else - _CheckReplicaPathsRemote "${TARGET[$__replicaDir]}" & + _CheckReplicasRemote "${TARGET[$__replicaDir]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? -ne 0 ]; then Logger "Cancelling task." "CRITICAL" exit 1 fi } -function _CheckDiskSpaceLocal { - local replica_path="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG +function _HandleLocksLocal { + local replicaStateDir="${1}" + local lockfile="${2}" + local replicaType="${3}" + local overwrite="${4:-false}" - local diskSpace - - Logger "Checking minimum disk space in [$replica_path]." "NOTICE" - - diskSpace=$($DF_CMD "$replica_path" | tail -1 | awk '{print $4}') - - if [ $? != 0 ]; then - Logger "Cannot get free space." "ERROR" - else - # Ugly fix for df in some busybox environments that can only show human formats - if [ $(IsInteger $diskSpace) -eq 0 ]; then - diskSpace=$(HumanToNumeric $diskSpace) - fi - - if [ $diskSpace -lt $MINIMUM_SPACE ]; then - Logger "There is not enough free space on replica [$replica_path] ($diskSpace KB)." "WARN" - fi - fi -} - -function _CheckDiskSpaceRemote { - local replica_path="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - Logger "Checking remote minimum disk space in [$replica_path]." "NOTICE" - - local cmd - local diskSpace - - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - - cmd=$SSH_CMD' "'$COMMAND_SUDO' '$DF_CMD' \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - if [ $? != 0 ]; then - Logger "Cannot get free space on target [$replica_path]." "ERROR" - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" - else - diskSpace=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID | tail -1 | awk '{print $4}') - - # Ugly fix for df in some busybox environments that can only show human formats - if [ $(IsInteger $diskSpace) -eq 0 ]; then - diskSpace=$(HumanToNumeric $diskSpace) - fi - - if [ $diskSpace -lt $MINIMUM_SPACE ]; then - Logger "There is not enough free space on replica [$replica_path] ($diskSpace KB)." "WARN" - fi - fi -} - -function CheckDiskSpace { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - local pids - - if [ $MINIMUM_SPACE -eq 0 ]; then - Logger "Skipped minimum space check." "NOTICE" - return 0 - fi - - _CheckDiskSpaceLocal "${INITIATOR[$__replicaDir]}" & - pids="$!" - if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckDiskSpaceLocal "${TARGET[$__replicaDir]}" & - pids="$pids;$!" - else - _CheckDiskSpaceRemote "${TARGET[$__replicaDir]}" & - pids="$pids;$!" - fi - WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} -} - - -function _CreateStateDirsLocal { - local replica_state_dir="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - if ! [ -d "$replica_state_dir" ]; then - $COMMAND_SUDO mkdir -p "$replica_state_dir" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 - if [ $? != 0 ]; then - Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL" - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" - exit 1 - fi - fi -} - -function _CreateStateDirsRemote { - local replica_state_dir="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - local cmd - - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - - cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - if [ $? != 0 ]; then - Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL" - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" - exit 1 - fi -} - -function CreateStateDirs { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - local pids - - _CreateStateDirsLocal "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}" & - pids="$!" - if [ "$REMOTE_OPERATION" != "yes" ]; then - _CreateStateDirsLocal "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" & - pids="$pids;$!" - else - _CreateStateDirsRemote "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" & - pids="$pids;$!" - fi - WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} - if [ $? -ne 0 ]; then - Logger "Cancelling task." "CRITICAL" - exit 1 - fi -} - -function _CheckLocksLocal { - local lockfile="${1}" - local replicaType="${2}" - - __CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local lockfileContent local lockPid local lockInstanceID + local writeLocks - if [ -s "$lockfile" ]; then + if [ ! -d "$replicaStateDir" ]; then + $COMMAND_SUDO mkdir -p "$replicaStateDir" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 + if [ $? != 0 ]; then + Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" + return 1 + fi + fi + + # Skip the whole part if overwrite true + if [ -s "$lockfile" ] && [ $overwrite != true ]; then lockfileContent=$(cat $lockfile) Logger "Master lock pid present: $lockfileContent" "DEBUG" - lockPid=${lockfileContent%@*} + lockPid="${lockfileContent%@*}" if [ $(IsInteger $lockPid) -ne 1 ]; then Logger "Invalid pid [$lockPid] in local replica." "CRITICAL" - exit 1 + return 1 fi - lockInstanceID=${lockfileContent#*@} + lockInstanceID="${lockfileContent#*@}" if [ "$lockInstanceID" == "" ]; then Logger "Invalid instance id [$lockInstanceID] in local replica." "CRITICAL" - exit 1 + return 1 Logger "Local $replicaType lock is: [$lockPid@$lockInstanceID]." "DEBUG" @@ -436,80 +391,145 @@ function _CheckLocksLocal { kill -0 $lockPid > /dev/null 2>&1 if [ $? != 0 ]; then Logger "There is a local dead osync lock [$lockPid@$lockInstanceID] that is no longer running. Resuming." "NOTICE" - if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then - # REPLICA_OVERWRITE_LOCK disables noclobber option in WriteLock functions - INITIATOR_OVERWRITE_LOCK=true - elif [ "$replicaType" == "${TARGET[$__type]}" ]; then - TARGET_OVERWRITE_LOCK=true - fi + writeLocks=true else Logger "There is already a local instance [$lockPid@$lockInstanceID] of osync running for this replica. Cannot start." "CRITICAL" - exit 1 + return 1 + fi + else + writeLocks=true + fi + + if [ $writeLocks != true ]; then + Logger "This is the final merdier" "WARN" + return 1 + else + $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID" + if [ $? != 0 ]; then + Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" + Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID)" "NOTICE" + return 1 + else + Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" fi fi } -function _CheckLocksRemote { - local lockfile="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG +function _HandleLocksRemote { + local replicaStateDir="${1}" + local lockfile="${2}" + local replicaType="${3}" + local overwrite="${4:-false}" - local cmd - local lockPid - local lockInstanceID - local lockfileContent + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG + + local initiatorRunningPids CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'"' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - if [ $? != 0 ]; then - Logger "Cannot check remote replica lock." "CRITICAL" - exit 1 + # Create an array of all currently running pids + # TODO: check portability + read -a initiatorRunningPids <<< $(ps -A | tail -n +2 | awk '{print $1}') + +# passing initiatorRunningPids as litteral string (has to be run through eval to be an array again) +$SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ +env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" env SCRIPT_PID="'$SCRIPT_PID'" \ +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 +include #### DEBUG SUBSET #### +include #### TrapError SUBSET #### +include #### ArrayContains SUBSET #### +include #### IsInteger SUBSET #### +include #### RemoteLogger SUBSET #### + +function _HandleLocksRemoteSub { + #WIP do not remote log to file as output is already logged from ssh + if [ ! -d "$replicaStateDir" ]; then + $COMMAND_SUDO mkdir -p "$replicaStateDir" + if [ $? != 0 ]; then + RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" + return 1 + fi fi - if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then - lockfileContent="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" - - lockPid=${lockfileContent%@*} + # Skip the whole part if overwrite true + if [ -s "$lockfile" ] && [ $overwrite != true ]; then + lockfileContent=$(cat $lockfile) + RemoteLogger "Master lock pid present: $lockfileContent" "DEBUG" + lockPid="${lockfileContent%@*}" if [ $(IsInteger $lockPid) -ne 1 ]; then - Logger "Invalid pid [$lockPid] in remote replica lock." "CRITICAL" - exit 1 + RemoteLogger "Invalid pid [$lockPid] in local replica." "CRITICAL" + return 1 fi - lockInstanceID=${lockfileContent#*@} + lockInstanceID="${lockfileContent#*@}" if [ "$lockInstanceID" == "" ]; then - Logger "Invalid instance id [$lockInstanceID] in remote replica." "CRITICAL" - exit 1 + RemoteLogger "Invalid instance id [$lockInstanceID] in local replica." "CRITICAL" + return 1 + + RemoteLogger "Local $replicaType lock is: [$lockPid@$lockInstanceID]." "DEBUG" + fi - Logger "Remote lock is: [$lockPid@$lockInstanceID]" "DEBUG" - - kill -0 $lockPid > /dev/null 2>&1 - if [ $? != 0 ]; then + # Retransform litteral array string to array + eval "initiatorRunningPids=$initiatorRunningPidsFlat" + if [ $(ArrayContains "$lockPid" "${initiatorRunningPids[@]}") -eq 0 ]; then if [ "$lockInstanceID" == "$INSTANCE_ID" ]; then - Logger "There is a remote dead osync lock [$lockPid@lockInstanceID] on target replica that corresponds to this initiator INSTANCE_ID. Pid [$lockPid] no longer running. Resuming." "NOTICE" - TARGET_OVERWRITE_LOCK=true + RemoteLogger "There is a remote dead osync lock [$lockPid@$lockInstanceID] on target replica that corresponds to this initiator INSTANCE_ID. Pid [$lockPid] no longer running. Resuming." "NOTICE" + writeLocks=true else if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]; then - Logger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Forcing resume." "WARN" - TARGET_OVERWRITE_LOCK=true + RemoteLogger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Forcing resume." "WARN" + writeLocks=true else - Logger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Will not resume." "CRITICAL" - exit 1 + RemoteLogger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Will not resume." "CRITICAL" + return 1 fi fi else - Logger "There is already a local instance of osync that locks target replica [$lockPid@$lockInstanceID]. Cannot start." "CRITICAL" - exit 1 + RemoteLogger "There is already a local instance of osync that locks target replica [$lockPid@$lockInstanceID]. Cannot start." "CRITICAL" + return 1 + fi + else + writeLocks=true + fi + + if [ $writeLocks != true ]; then + return 1 + else + $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + if [ $? != 0 ]; then + RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" + return 1 + else + RemoteLogger "Locked local $replicaType replica in [$lockfile]." "DEBUG" fi fi } -function CheckLocks { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG +_HandleLocksRemoteSub +result=$? +exit $result +ENDSSH + + if [ $? != 0 ]; then + Logger "Remote lock handling failed." "CRITICAL" + fi + if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then + ( + _LOGGER_PREFIX="" + Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" + ) + fi + if [ $? != 0 ]; then + return 1 + fi +} + +function HandleLocks { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids + local overwrite=false if [ $_NOLOCKS == true ]; then return 0 @@ -517,116 +537,40 @@ function CheckLocks { # Do not bother checking for locks when FORCE_UNLOCK is set if [ $FORCE_UNLOCK == true ]; then - WriteLockFiles - if [ $? != 0 ]; then - exit 1 - fi + overwrite=true else - _CheckLocksLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" & + _HandleLocksLocal "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}" "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" $overwrite & pids="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _CheckLocksLocal "${TARGET[$__lockFile]}" "${INITIATOR[$__type]}" & + _HandleLocksLocal "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $overwrite & pids="$pids;$!" else - _CheckLocksRemote "${TARGET[$__lockFile]}" & + _HandleLocksRemote "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $overwrite & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + INITIATOR_LOCK_FILE_EXISTS=true + TARGET_LOCK_FILE_EXISTS=true + WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? -ne 0 ]; then - Logger "Cancelling task." "CRITICAL" - exit 1 + IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" + for pid in "${pidArray[@]}"; do + pid=${pid%:*} + if [ "$pid" == "$initiatorPid" ]; then + INITIATOR_LOCK_FILE_EXISTS=false + elif [ "$pid" == "$targetPid" ]; then + TARGET_LOCK_FILE_EXISTS=false + fi + done + + Logger "Cancelling task." "CRITICAL" + exit 1 fi - WriteLockFiles - fi -} - -function _WriteLockFilesLocal { - local lockfile="${1}" - local replicaType="${2}" - local overwrite="${3:-false}" - - __CheckArguments 2-3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - ( - if [ $overwrite == true ]; then - set -o noclobber - fi - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID" - ) - if [ $? != 0 ]; then - Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" - Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID)" "NOTICE" - return 1 - else - Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" - fi -} - -function _WriteLockFilesRemote { - local lockfile="${1}" - local replicaType="${2}" - local overwrite="${3-false}" - - __CheckArguments 2-3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - local cmd - - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - - cmd=$SSH_CMD' "( if [ $overwrite == true ]; then set -o noclobber; fi; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - if [ $? != 0 ]; then - Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL" - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID)" "NOTICE" - return 1 - else - Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG" - fi -} - -function WriteLockFiles { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG - - local initiatorPid - local targetPid - local pidArray - local pid - - _WriteLockFilesLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" $INITIATOR_LOCK_OVERWRITE & - initiatorPid="$!" - - if [ "$REMOTE_OPERATION" != "yes" ]; then - _WriteLockFilesLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $TARGET_LOCK_OVERWRITE & - targetPid="$!" - else - _WriteLockFilesRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $TARGET_LOCK_OVERWRITE & - targetPid="$!" - fi - - INITIATOR_LOCK_FILE_EXISTS=true - TARGET_LOCK_FILE_EXISTS=true - WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} - if [ $? -ne 0 ]; then - IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" - for pid in "${pidArray[@]}"; do - pid=${pid%:*} - if [ "$pid" == "$initiatorPid" ]; then - INITIATOR_LOCK_FILE_EXISTS=false - elif [ "$pid" == "$targetPid" ]; then - TARGET_LOCK_FILE_EXISTS=false - fi - done - - Logger "Cancelling task." "CRITICAL" - exit 1 fi } function _UnlockReplicasLocal { local lockfile="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG if [ -f "$lockfile" ]; then $COMMAND_SUDO rm "$lockfile" @@ -640,16 +584,19 @@ function _UnlockReplicasLocal { function _UnlockReplicasRemote { local lockfile="${1}" - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local cmd= CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" +#TODO: missing logger entry +$SSH_CMD env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 +if [ -f "$lockfile" ]; then + $COMMAND_SUDO rm -f "$lockfile" +fi +ENDSSH if [ $? != 0 ]; then Logger "Could not unlock remote replica." "ERROR" Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" @@ -659,7 +606,7 @@ function _UnlockReplicasRemote { } function UnlockReplicas { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -683,7 +630,7 @@ function UnlockReplicas { fi if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false fi } @@ -699,7 +646,7 @@ function treeList { local replicaType="${2}" # replica type: initiator, target local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) - __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local escapedReplicaPath local rsyncCmd @@ -747,7 +694,7 @@ function treeList { # deleteList(replicaType): Creates a list of files vanished from last run on replica $1 (initiator/target) function deleteList { local replicaType="${1}" # replica type: initiator, target - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local cmd @@ -796,7 +743,7 @@ function _getFileCtimeMtimeLocal { local replicaPath="${1}" # Contains replica path local replicaType="${2}" # Initiator / Target local fileList="${3}" # Contains list of files to get time attrs - __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" while read -r file; do $STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID"; done < "$fileList" @@ -805,7 +752,7 @@ function _getFileCtimeMtimeLocal { if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" ]; then Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID)" "VERBOSE" fi - exit 1 + return 1 fi } @@ -814,19 +761,39 @@ function _getFileCtimeMtimeRemote { local replicaPath="${1}" # Contains replica path local replicaType="${2}" local fileList="${3}" - __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local cmd - cmd='cat "'$fileList'" | '$SSH_CMD' "while read -r file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replicaPath'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID'"' + #TODO WIP + # Quoting ' in single quote with '"'"' in order to have cmd='some stuff \'bash -c "some other stuff"\'' + #cmd='cat "'$fileList'" | '$SSH_CMD' '"'"'bash -c "while read -r file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replicaPath'\$file\"; done | sort"'"'"' > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID'"' + cmd='cat "'$fileList'" | '$SSH_CMD' "cat > \".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID\""' Logger "CMD: $cmd" "DEBUG" eval "$cmd" + if [ $? != 0 ]; then + Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" + if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID)" "VERBOSE" + fi + return 1 + fi + +$SSH_CMD env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" + while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" + if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" ]; then + rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" + fi +ENDSSH if [ $? != 0 ]; then Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" ]; then Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID)" "VERBOSE" fi - exit 1 + return 1 + else + # Ugly fix for csh in FreeBSD 11 that adds leading and trailing '\"' + sed -i.tmp -e 's/^\\"//' -e 's/\\"$//' "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID" fi } @@ -835,7 +802,7 @@ function _getFileCtimeMtimeRemote { function syncAttrs { local initiatorReplica="${1}" local targetReplica="${2}" - __CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local rsyncCmd local retval @@ -862,7 +829,7 @@ function syncAttrs { fi Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false retval=$? if [ $retval != 0 ] && [ $retval != 24 ]; then @@ -870,7 +837,7 @@ function syncAttrs { if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi - exit 1 + return 1 else if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE" @@ -894,7 +861,11 @@ function syncAttrs { _getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" & pids="$pids;$!" fi - WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false + if [ $? != 0 ]; then + Logger "Getting ctime attributes failed." "CRITICAL" + return 1 + fi # If target gets updated first, then sync_attr must update initiators attrs first # For join, remove leading replica paths @@ -942,7 +913,7 @@ function syncAttrs { Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false retval=$? if [ $retval != 0 ] && [ $retval != 24 ]; then @@ -950,7 +921,7 @@ function syncAttrs { if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID)" "NOTICE" fi - exit 1 + return 1 else if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID" ]; then Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID)" "VERBOSE" @@ -963,7 +934,7 @@ function syncAttrs { function syncUpdate { local sourceReplica="${1}" # Contains replica type of source: initiator, target local destinationReplica="${2}" # Contains replica type of destination: initiator, target - __CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local rsyncCmd local retval @@ -1026,7 +997,7 @@ function _deleteLocal { local replicaType="${1}" # Replica type local replicaDir="${2}" # Full path to replica local deletionDir="${3}" # deletion dir in format .[workdir]/deleted - __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local parentdir local previousFile="" @@ -1103,7 +1074,7 @@ function _deleteRemote { local replicaType="${1}" # Replica type local replicaDir="${2}" # Full path to replica local deletionDir="${3}" # deletion dir in format .[workdir]/deleted - __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local escDestDir local rsyncCmd @@ -1113,8 +1084,6 @@ function _deleteRemote { local deletionListFromReplica - local loggerPrefix - if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then deletionListFromReplica="${TARGET[$__type]}" elif [ "$replicaType" == "${TARGET[$__type]}" ]; then @@ -1143,47 +1112,13 @@ function _deleteRemote { exit 1 fi -$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _LOGGER_VERBOSE=$_LOGGER_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETE_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 - - ## The following lines are executed remotely - function _logger { - local value="${1}" # What to log - echo -e "$value" - } - - function Logger { - local value="${1}" # What to log - local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL - - local prefix="RTIME: $SECONDS - " - - if [ "$level" == "CRITICAL" ]; then - _logger "$prefix\e[41m$value\e[0m" - return - elif [ "$level" == "ERROR" ]; then - _logger "$prefix\e[91m$value\e[0m" - return - elif [ "$level" == "WARN" ]; then - _logger "$prefix\e[93m$value\e[0m" - return - elif [ "$level" == "NOTICE" ]; then - _logger "$prefix$value" - return - elif [ "$level" == "VERBOSE" ]; then - if [ $_LOGGER_VERBOSE == true ]; then - _logger "$prefix$value" - fi - return - elif [ "$level" == "DEBUG" ]; then - if [ "$_DEBUG" == "yes" ]; then - _logger "$prefix$value" - fi - return - else - _logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" - _logger "$prefix$value" - fi - } +$SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 +include #### DEBUG SUBSET #### +include #### TrapError SUBSET #### +include #### RemoteLogger SUBSET #### ## Empty earlier failed delete list > "$FAILED_DELETE_LIST" @@ -1195,7 +1130,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$ if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" if [ $? != 0 ]; then - Logger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" + RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" exit 1 fi fi @@ -1214,15 +1149,15 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$ # In order to keep full path on soft deletion, create parent directories before move parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then - Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" + RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else - Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" + RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi if [ $? != 0 ]; then - Logger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR" + RemoteLogger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR" # Using $files instead of $REPLICA_DIR$files here so the list is ready for next run echo "$files" >> "$FAILED_DELETE_LIST" else @@ -1233,10 +1168,10 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$ else if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then - Logger "Deleting [$REPLICA_DIR$files]." "VERBOSE" + RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" if [ $? != 0 ]; then - Logger "Cannot delete [$REPLICA_DIR$files]." "ERROR" + RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" echo "$files" >> "$FAILED_DELETE_LIST" else echo "$files" >> "$SUCCESS_DELETE_LIST" @@ -1250,10 +1185,10 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$ ENDSSH if [ -s "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then - loggerPrefix="$_LOGGER_PREFIX" + ( _LOGGER_PREFIX="" Logger "$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "ERROR" - _LOGGER_PREFIX="$loggerPrefix" + ) fi ## Copy back the deleted failed file list @@ -1275,7 +1210,7 @@ ENDSSH # delete_Propagation(replica type) function deletionPropagation { local replicaType="${1}" # Contains replica type: initiator, target where to delete - __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local replicaDir local deleteDir @@ -1283,7 +1218,7 @@ function deletionPropagation { Logger "Propagating deletions to $replicaType replica." "NOTICE" if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then - if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then + if [ $(ArrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then replicaDir="${INITIATOR[$__replicaDir]}" deleteDir="${INITIATOR[$__deleteDir]}" @@ -1297,7 +1232,7 @@ function deletionPropagation { Logger "Skipping deletion on replica $replicaType." "NOTICE" fi elif [ "$replicaType" == "${TARGET[$__type]}" ]; then - if [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then + if [ $(ArrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then replicaDir="${TARGET[$__replicaDir]}" deleteDir="${TARGET[$__deleteDir]}" @@ -1335,7 +1270,7 @@ function deletionPropagation { ###### Step 5a & 5b: Create after run file list of replicas function Sync { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local resumeCount local resumeInitiator @@ -1408,7 +1343,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" initiatorFail=false @@ -1453,7 +1388,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" initiatorFail=false @@ -1490,7 +1425,7 @@ function Sync { if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}" @@ -1515,7 +1450,7 @@ function Sync { if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" resumeTarget="${SYNC_ACTION[3]}" @@ -1527,7 +1462,7 @@ function Sync { fi if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" resumeInitiator="${SYNC_ACTION[3]}" @@ -1540,7 +1475,7 @@ function Sync { else if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" resumeInitiator="${SYNC_ACTION[3]}" @@ -1552,7 +1487,7 @@ function Sync { fi if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" resumeTarget="${SYNC_ACTION[3]}" @@ -1577,7 +1512,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" initiatorFail=false @@ -1623,7 +1558,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" initiatorFail=false @@ -1666,7 +1601,7 @@ function _SoftDeleteLocal { local changeTime="${3}" # Delete files older than changeTime days local deletionType="${4}" # Trivial deletion type string - __CheckArguments 4 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -1715,7 +1650,7 @@ function _SoftDeleteRemote { local changeTime="${3}" # Delete files older than changeTime days local deletionType="${4}" # Trivial deletion type string - __CheckArguments 4 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -1733,22 +1668,18 @@ function _SoftDeleteRemote { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - if [ $_LOGGER_VERBOSE == true ]; then - # Cannot launch log function from xargs, ugly hack - cmd=$SSH_CMD' "if [ -d \"'$replicaDeletionPath'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type f -ctime +'$changeTime' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type d -empty -ctime '$changeTime' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE" - fi + #TODO: fixed find directory -ctime was missing +, test it +#TODO: missing logger vars +$SSH_CMD env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _DRYRUN="'$_DRYRUN'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 - if [ $_DRYRUN == false ]; then - cmd=$SSH_CMD' "if [ -d \"'$replicaDeletionPath'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type f -ctime +'$changeTime' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type d -empty -ctime '$changeTime' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"The $replicaType replicaDir [$replicaDeletionPath] does not exist. Skipping cleaning of old files.\"; fi" >> "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' - - Logger "cmd: $cmd" "DEBUG" - eval "$cmd" - else - Dummy - fi +# Cannot launch log function from xargs, ugly hack +if [ -d "$replicaDeletionPath" ]; then +$COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "Will delete file {}"; if [ '$DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"' +$COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "Will delete directory {}"; if [ '$DRY_RUN' == false ]; then '$COMMAND_SUDO' rm -f "{}"' +else + echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" +fi +ENDSSH retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" @@ -1759,7 +1690,7 @@ function _SoftDeleteRemote { } function SoftDelete { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1775,7 +1706,7 @@ function SoftDelete { _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" & pids="$pids;$!" fi - WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then exit 1 fi @@ -1793,7 +1724,7 @@ function SoftDelete { _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" & pids="$pids;$!" fi - WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]} + WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? != 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then exit 1 fi @@ -1805,7 +1736,7 @@ function _SummaryFromFile { local summaryFile="${2}" local direction="${3}" - __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG if [ -f "$summaryFile" ]; then while read -r file; do @@ -1815,11 +1746,9 @@ function _SummaryFromFile { } function Summary { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG - local loggerPrefix - - loggerPrefix="$_LOGGER_PREFIX" + ( _LOGGER_PREFIX="" Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS" @@ -1838,12 +1767,11 @@ function Summary { _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID" "- >>" fi _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "- <<" - - _LOGGER_PREFIX="$loggerPrefix" + ) } function Init { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Set error exit code if a piped command fails set -o pipefail @@ -2025,15 +1953,14 @@ function Init { } function Main { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG - CreateStateDirs - CheckLocks + HandleLocks Sync } function Usage { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$IS_STABLE" != "yes" ]; then echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" @@ -2076,7 +2003,7 @@ function Usage { } function SyncOnChanges { - __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local cmd local retval @@ -2112,11 +2039,11 @@ function SyncOnChanges { Logger "#### Monitoring now." "NOTICE" if [ "$LOCAL_OS" == "MacOSX" ]; then - fswatch --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -1 "$INITIATOR_SYNC_DIR" > /dev/null & + fswatch $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null & # Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout - WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true ${FUNCNAME[0]} + WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true else - inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & + inotifywait $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & wait $! fi retval=$? @@ -2340,8 +2267,7 @@ else SOFT_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0 fi - CheckReplicaPaths - CheckDiskSpace + CheckReplicas RunBeforeHook Main if [ $? == 0 ]; then diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 830cf52..d12508a 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,7 +1,9 @@ -#### MINIMAL-FUNCTION-SET BEGIN #### +#### OFUNCTIONS FULL SUBSET #### +#### OFUNCTIONS MINI SUBSET #### -_OFUNCTIONS_VERSION=2.0 -_OFUNCTIONS_BUILD=2016120401 +_OFUNCTIONS_VERSION=2.1 +_OFUNCTIONS_BUILD=2016120901 +_OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: @@ -41,6 +43,7 @@ fi ERROR_ALERT=false WARN_ALERT=false +#### DEBUG SUBSET #### ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG @@ -58,6 +61,7 @@ fi if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console SLEEP_TIME=.05 fi +#### DEBUG SUBSET END #### SCRIPT_PID=$$ @@ -73,8 +77,10 @@ if [ -w /var/log ]; then LOG_FILE="/var/log/$PROGRAM.log" elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then LOG_FILE="$HOME/$PROGRAM.log" -else +elif [ -w . ]; then LOG_FILE="./$PROGRAM.log" +else + LOG_FILE="/tmp/$PROGRAM.log" fi ## Default directory where to store temporary run files @@ -96,20 +102,24 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.last.log" function Dummy { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG sleep $SLEEP_TIME } +#### Logger SUBSET #### +#### RemoteLogger SUBSET #### # Sub function of Logger function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen local toStderr="${3:-false}" # Log to stderr instead of stdout - echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + if [ "$logValue" != "" ]; then + echo -e "$logValue" >> "$LOG_FILE" + # Current log file + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then if [ $toStderr == true ]; then @@ -122,6 +132,58 @@ function _Logger { fi } +# Remote logger similar to below Logger, without log to file and alert flags +function RemoteLogger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level + + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then + prefix="$(date) - " + else + prefix="" + fi + + if [ "$level" == "CRITICAL" ]; then + _Logger "" "$prefix\e[41m$value\e[0m" true + return + elif [ "$level" == "ERROR" ]; then + _Logger "" "$prefix\e[91m$value\e[0m" true + return + elif [ "$level" == "WARN" ]; then + _Logger "" "$prefix\e[33m$value\e[0m" true + return + elif [ "$level" == "NOTICE" ]; then + if [ $_LOGGER_ERR_ONLY != true ]; then + _Logger "" "$prefix$value" + fi + return + elif [ "$level" == "VERBOSE" ]; then + if [ $_LOGGER_VERBOSE == true ]; then + _Logger "" "$prefix$value" + fi + return + elif [ "$level" == "ALWAYS" ]; then + _Logger "" "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "" "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" + _Logger "Value was: $prefix$value" + fi +} +#### RemoteLogger SUBSET END #### + # General log function with log levels: # Environment variables @@ -193,13 +255,14 @@ function Logger { _Logger "Value was: $prefix$value" fi } +#### Logger SUBSET END #### # QuickLogger subfunction, can be called directly function _QuickLogger { local value="${1}" local destination="${2}" # Destination: stdout, log, both - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then echo -e "$(date) - $value" >> "$LOG_FILE" @@ -212,7 +275,7 @@ function _QuickLogger { function QuickLogger { local value="${1}" - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG if [ $_LOGGER_SILENT == true ]; then _QuickLogger "$value" "log" @@ -226,7 +289,7 @@ function KillChilds { local pid="${1}" # Parent pid to kill childs local self="${2:-false}" # Should parent be killed too ? - + # Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then for child in $children; do Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG @@ -261,7 +324,7 @@ function KillAllChilds { local pids="${1}" # List of parent pids to kill separated by semi-colon local self="${2:-false}" # Should parent be killed too ? - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local errorcount=0 @@ -279,7 +342,7 @@ function KillAllChilds { function SendAlert { local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run - __CheckArguments 0-1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0-1 $# "$@" #__WITH_PARANOIA_DEBUG local attachment local attachmentFile @@ -362,7 +425,7 @@ function SendEmail { local smtpUser="${9}" local smtpPassword="${10}" - __CheckArguments 3-10 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3-10 $# "$@" #__WITH_PARANOIA_DEBUG local mail_no_attachment= local attachment_command= @@ -499,6 +562,7 @@ function SendEmail { Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue } +#### TrapError SUBSET #### function TrapError { local job="$0" local line="$1" @@ -508,11 +572,12 @@ function TrapError { echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" fi } +#### TrapError SUBSET END #### function LoadConfigFile { local configFile="${1}" - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG if [ ! -f "$configFile" ]; then @@ -573,7 +638,7 @@ function joinString { # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 # Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached -# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false "${FUNCNAME[0]}" +# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon @@ -584,10 +649,10 @@ function WaitForTaskCompletion { local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false) local spinner="${7:-true}" # Show spinner (true), don't show anything (false) local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true) - local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]} + local callerName="${FUNCNAME[1]}" Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - __CheckArguments 9 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 8 $# "$@" #__WITH_PARANOIA_DEBUG local log_ttime=0 # local time instance for comparaison @@ -730,9 +795,9 @@ function ParallelExec { local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false) local spinner="${9:-false}" # Show spinner (true), don't show spinner (false) local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true) - local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]} - __CheckArguments 2-11 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local callerName="${FUNCNAME[1]}" + __CheckArguments 2-10 $# "$@" #__WITH_PARANOIA_DEBUG local log_ttime=0 # local time instance for comparaison @@ -871,7 +936,7 @@ function ParallelExec { } function CleanUp { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$_DEBUG" != "yes" ]; then rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID" @@ -939,6 +1004,7 @@ function IsNumeric { fi } +#### IsInteger SUBSET #### function IsInteger { local value="${1}" @@ -948,7 +1014,9 @@ function IsInteger { echo 0 fi } +#### IsInteger SUBSET END #### +#### HumanToNumeric SUBSET #### # Converts human readable sizes into integer kilobyte sizes # Usage numericSize="$(HumanToNumeric $humanSize)" function HumanToNumeric { @@ -979,9 +1047,10 @@ function HumanToNumeric { echo $value } +#### HumanToNumeric SUBSET END #### ## from https://gist.github.com/cdown/1163649 -function urlEncode { +function UrlEncode { local length="${#1}" local LANG=C @@ -998,29 +1067,35 @@ function urlEncode { done } -function urlDecode { +function UrlDecode { local urlEncoded="${1//+/ }" printf '%b' "${urlEncoded//%/\\x}" } +#### ArrayContains SUBSET #### ## Modified version of http://stackoverflow.com/a/8574392 -## Usage: arrayContains "needle" "${haystack[@]}" -arrayContains () { +## Usage: [ $(ArrayContains "needle" "${haystack[@]}") -eq 1 ] +function ArrayContains () { + local needle="${1}" + local haystack="${2}" local e - if [ "$2" == "" ]; then - echo 0 && return 0 + if [ "$needle" != "" ] && [ "$haystack" != "" ]; then + for e in "${@:2}"; do + if [ "$e" == "$needle" ]; then + echo 1 + return + fi + done fi - - for e in "${@:2}"; do - [[ "$e" == "$1" ]] && echo 1 && return 1 - done - echo 0 && return 0 + echo 0 + return } +#### ArrayContains SUBSET END #### function GetLocalOS { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local localOsVar @@ -1053,9 +1128,12 @@ function GetLocalOS { *"BSD"*) LOCAL_OS="BSD" ;; - *"MINGW32"*|*"CYGWIN"*) + *"MINGW32"*|*"MSYS"*) LOCAL_OS="msys" ;; + *"CYGWIN"*) + LOCAL_OS="Cygwin" + ;; *"Microsoft"*) LOCAL_OS="WinNT10" ;; @@ -1077,10 +1155,10 @@ function GetLocalOS { Logger "Local OS: [$localOsVar]." "DEBUG" } -#### MINIMAL-FUNCTION-SET END #### +#### OFUNCTIONS MINI SUBSET END #### function GetRemoteOS { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != "yes" ]; then return 0 @@ -1129,9 +1207,12 @@ ENDSSH *"BSD"*) REMOTE_OS="BSD" ;; - *"MINGW32"*|*"CYGWIN"*) + *"MINGW32"*|*"MSYS"*) REMOTE_OS="msys" ;; + *"CYGWIN"*) + REMOTE_OS="Cygwin" + ;; *"Microsoft"*) REMOTE_OS="WinNT10" ;; @@ -1163,7 +1244,7 @@ ENDSSH function RunLocalCommand { local command="${1}" # Command to run local hardMaxTime="${2}" # Max time to wait for command to compleet - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG if [ $_DRYRUN == true ]; then Logger "Dryrun: Local command [$command] not run." "NOTICE" @@ -1195,7 +1276,7 @@ function RunLocalCommand { function RunRemoteCommand { local command="${1}" # Command to run local hardMaxTime="${2}" # Max time to wait for command to compleet - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -1228,7 +1309,7 @@ function RunRemoteCommand { } function RunBeforeHook { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1247,7 +1328,7 @@ function RunBeforeHook { } function RunAfterHook { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1266,7 +1347,7 @@ function RunAfterHook { } function CheckConnectivityRemoteHost { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -1285,7 +1366,7 @@ function CheckConnectivityRemoteHost { } function CheckConnectivity3rdPartyHosts { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local remote3rdPartySuccess local retval @@ -1323,16 +1404,17 @@ function __CheckArguments { if [ "$_DEBUG" == "yes" ]; then local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments local numberOfGivenArguments="${2}" # Number of arguments that have been passed - local functionName="${3}" # Function name that called __CheckArguments local minArgs local maxArgs - # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) + # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) # If any of the arguments contains spaces, bash things there are two aguments - # In order to avoid this, we need to iterate over ${4} and count + # In order to avoid this, we need to iterate over ${3} and count - local iterate=4 + callerName="${FUNCNAME[1]}" + + local iterate=3 local fetchArguments=true local argList="" local countedArguments @@ -1342,12 +1424,12 @@ function __CheckArguments { if [ "$argument" = "" ]; then fetchArguments=false else - argList="$argList[Argument $(($iterate-3)): $argument] " + argList="$argList[Argument $(($iterate-2)): $argument] " iterate=$(($iterate+1)) fi done - countedArguments=$((iterate-4)) + countedArguments=$((iterate-3)) if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then minArgs=$numberOfArguments @@ -1356,13 +1438,15 @@ function __CheckArguments { IFS='-' read minArgs maxArgs <<< "$numberOfArguments" fi - Logger "Entering function [$functionName]." "PARANOIA_DEBUG" + Logger "Entering function [$callerName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" - Logger "Arguments passed: $argList" "ERROR" + Logger "$callerName arguments: $argList" "ERROR" else - Logger "Arguments passed: $argList" "PARANOIA_DEBUG" + if [ ! -z "$argList" ]; then + Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" + fi fi fi } @@ -1372,7 +1456,7 @@ function __CheckArguments { function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local rest @@ -1402,7 +1486,7 @@ function RsyncPatternsAdd { function RsyncPatternsFromAdd { local patternType="${1}" local patternFrom="${2}" - __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename $patternFrom)" == "$patternFrom" ]; then @@ -1415,7 +1499,7 @@ function RsyncPatternsFromAdd { } function RsyncPatterns { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then @@ -1450,7 +1534,7 @@ function RsyncPatterns { } function PreInit { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local compressionString @@ -1495,7 +1579,7 @@ function PreInit { } function PostInit { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then @@ -1515,13 +1599,13 @@ function PostInit { } function InitLocalOSDependingSettings { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG ## If running under Msys, some commands do not run the same way ## Using mingw version of find instead of windows one ## Getting running processes is quite different ## Ping command is not the same - if [ "$LOCAL_OS" == "msys" ]; then + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then FIND_CMD=$(dirname $BASH)/find PING_CMD='$SYSTEMROOT\system32\ping -n 2' else @@ -1529,7 +1613,7 @@ function InitLocalOSDependingSettings { PING_CMD="ping -c 2 -i .2" fi - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then PROCESS_STATE_CMD="echo none" DF_CMD="df" else @@ -1551,9 +1635,9 @@ function InitLocalOSDependingSettings { } function InitRemoteOSDependingSettings { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG - if [ "$REMOTE_OS" == "msys" ]; then + if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then REMOTE_FIND_CMD=$(dirname $BASH)/find else REMOTE_FIND_CMD=find @@ -1590,7 +1674,7 @@ function InitRemoteOSDependingSettings { if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability" fi - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ "$REMOTE_OS" != "MacOSX" ]; then if [ "$PRESERVE_ACL" == "yes" ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" fi @@ -1691,4 +1775,4 @@ function ParentPid { fi } -## END Generic functions +#### OFUNCTIONS FULL SUBSET END #### diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 3a85240..334aa05 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -7,7 +7,7 @@ ## On CYGWIN / MSYS, ACL and extended attributes aren't supported -# osync test suite 2016120802 +# osync test suite 2016120901 # 4 tests: # quicklocal @@ -221,7 +221,7 @@ function oneTimeSetUp () { osyncDaemonParameters[$__local]="$CONF_DIR/$LOCAL_CONF --on-changes" - if [ "$LOCAL_OS" != "msys" ]; then + if [ "$LOCAL_OS" != "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${HOME}/.ssh/id_rsa_local --instance-id=quickremote" osyncParameters[$__confRemote]="$CONF_DIR/$REMOTE_CONF" @@ -536,6 +536,11 @@ function test_handle_symlinks () { return 0 fi + if [ "$LOCAL_OS" == "msys" ]; then + echo "Skipping symlink tests because msys handles them strangely or not at all." + return 0 + fi + # Check with and without copySymlinks copySymlinks="no" @@ -735,7 +740,7 @@ function test_softdeletion_cleanup () { touch "$file.new" - if [ "$TRAVIS_RUN" == true ] || [ "$LOCAL_OS" == "BSD" ] || [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ]; then + if [ "$TRAVIS_RUN" == true ] || [ "$LOCAL_OS" == "BSD" ] || [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then echo "Skipping changing ctime on file because travis / bsd / macos / Win10 / msys / cygwin does not support debugfs" else CreateOldFile "$file.old" @@ -750,7 +755,7 @@ function test_softdeletion_cleanup () { [ -f "$file.new" ] assertEquals "New softdeleted / backed up file [$file.new] exists." "0" $? - if [ "$TRAVIS_RUN" == true ] || [ "$LOCAL_OS" == "BSD" ] || [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ]; then + if [ "$TRAVIS_RUN" == true ] || [ "$LOCAL_OS" == "BSD" ] || [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then [ ! -f "$file.old" ] assertEquals "Old softdeleted / backed up file [$file.old] is deleted permanently." "0" $? else @@ -1268,8 +1273,8 @@ function test_UpgradeConfRun () { } function test_DaemonMode () { - if [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ]; then - echo "Skipping daemon mode test as Win10 does not have inotifywait support." + if [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then + echo "Skipping daemon mode test as [$LOCAL_OS] does not have inotifywait support." return 0 fi