From 193aab025ec26e0ccdb7c9e08eee1c154d820a23 Mon Sep 17 00:00:00 2001 From: deajan Date: Sat, 13 Aug 2016 11:56:41 +0200 Subject: [PATCH] Rebuilt targets --- dev/debug_osync.sh | 102 ++++++++++++++++++--------------------------- install.sh | 2 +- osync.sh | 98 +++++++++++++++++-------------------------- 3 files changed, 79 insertions(+), 123 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 6121695..4800125 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -#TODO: handle conflict prevalance +#TODO(critical): handle conflict prevalance, especially in sync_attrs function PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-dev-parallel-unstable -PROGRAM_BUILD=2016080901 +PROGRAM_VERSION=1.2-dev-parallel +PROGRAM_BUILD=2016081301 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -35,8 +35,8 @@ IS_STABLE=no # _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG # sync_update yes #__WITH_PARANOIA_DEBUG # sync_update yes #__WITH_PARANOIA_DEBUG -# deletion_propagation yes? #__WITH_PARANOIA_DEBUG -# deletion_propagation yes? #__WITH_PARANOIA_DEBUG +# deletion_propagation yes #__WITH_PARANOIA_DEBUG +# deletion_propagation yes #__WITH_PARANOIA_DEBUG # tree_list yes #__WITH_PARANOIA_DEBUG # tree_list yes #__WITH_PARANOIA_DEBUG # SoftDelete yes #__WITH_PARANOIA_DEBUG @@ -46,12 +46,9 @@ IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016080903 +## FUNC_BUILD=2016081201 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr -#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode -#TODO: set _LOGGER_STDERR in other apps - ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" @@ -1325,9 +1322,11 @@ function InitRemoteOSSettings { #TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - #if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - # RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" - #fi + if [ "$PRESERVE_EXECUTABILITY" != "no" ];then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" + fi + fi if [ "$REMOTE_OS" == "msys" ]; then REMOTE_FIND_CMD=$(dirname $BASH)/find @@ -1461,7 +1460,7 @@ function CheckCurrentConfig { eval "$test" done - #TODO-v2.1: Add runtime variable tests (RSYNC_ARGS etc) + #TODO(low): Add runtime variable tests (RSYNC_ARGS etc) } ###### Osync specific functions (non shared) @@ -1524,7 +1523,7 @@ function CheckReplicaPaths { local pids - #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO: investigate realpath & readlink issues on MSYS and busybox here + #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO(verylow): investigate realpath & readlink issues on MSYS and busybox here #TARGET_SYNC_DIR_CANN=$(realpath "${TARGET[1]}) #if [ "$REMOTE_OPERATION" != "yes" ]; then @@ -2161,16 +2160,14 @@ function _delete_local { __CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local parentdir - - ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted local previous_file="" - #TODO: fix and add same to remote - if [ -d "$replica_dir$deletion_dir" ]; then + if [ -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN -ne 1 ]; then $COMMAND_SUDO mkdir -p "$replica_dir$deletion_dir" fi while read -r files; do + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [ $_VERBOSE -eq 1 ]; then Logger "Soft deleting $replica_dir$files" "NOTICE" @@ -2239,13 +2236,13 @@ function _delete_remote { exit 1 fi -#TODO: check output file without & and sleep 5 +#TODO(critical): check output file without & and sleep 5 $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 ## The following lines are executed remotely function _logger { local value="${1}" # What to log - echo -e "$value" >&2 #TODO logfile output missing + echo -e "$value" >&2 #TODO(high): logfile output missing if [ $_SILENT -eq 0 ]; then echo -e "$value" @@ -2282,10 +2279,14 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= > "$FAILED_DELETE_LIST" parentdir= - - ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted previous_file="" + + if [ -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN -ne 1 ]; then + $COMMAND_SUDO mkdir -p "$replica_dir$deletion_dir" + fi + while read -r files; do + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [ $_VERBOSE -eq 1 ]; then Logger "Soft deleting $REPLICA_DIR$files" "NOTICE" @@ -2360,11 +2361,7 @@ function deletion_propagation { replica_dir="${INITIATOR[1]}" delete_dir="${INITIATOR[5]}" - - #TODO: parallelize this ! - - _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" retval=$? if [ $retval != 0 ]; then Logger "Deletion on replica $replica_type failed." "CRITICAL" @@ -2375,11 +2372,10 @@ function deletion_propagation { delete_dir="${TARGET[5]}" if [ "$REMOTE_OPERATION" == "yes" ]; then - _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & + _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" else - _delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & + _delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false retval=$? if [ $retval == 0 ]; then if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE -eq 1 ]; then @@ -2396,15 +2392,14 @@ function deletion_propagation { fi } -###### Sync function in 6 steps (functions above) +###### Sync function in 6 steps ###### -###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S) -###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S) -###### Steps 3a & 3b can be skipped depending on $RSYNC_ATTR_ARGS is empty or not -###### Step 3a: Update initiator and target file attributes only -###### Step 3b: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence) -###### Step 4: Deleted file propagation to initiator and target replicas (Steps 4M and 4S) -###### Step 5: Create after run tree list for initiator and target replicas (Steps 5M and 5S) +###### Step 0a & 0b: Create current file list of replicas +###### Step 1a & 1b: Create deleted file list of replicas +###### Step 3: Update file attributes +###### Step 3a & 3b: Update replicas +###### Step 4a & 4b: Propagate deletions on replicas +###### Step 5a & 5b: Create after run file list of replicas function Sync { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -2456,7 +2451,6 @@ function Sync { fi else Logger "Will not resume aborted execution. Too many resume tries [$resumeCount]." "WARN" - #echo "noresume" > "${INITIATOR[7]}" echo "0" > "${INITIATOR[9]}" resumeInitiator="none" resumeTarget="none" @@ -2675,14 +2669,14 @@ function _SoftDeleteLocal { if [ $_VERBOSE -eq 1 ]; then # Cannot launch log function from xargs, ugly hack - $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" - $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi if [ $_DRYRUN -ne 1 ]; then - $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 + $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 else Dummy fi @@ -2717,14 +2711,14 @@ function _SoftDeleteRemote { if [ $_VERBOSE -eq 1 ]; then # Cannot launch log function from xargs, ugly hack - cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi if [ $_DRYRUN -ne 1 ]; then - cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" @@ -2845,6 +2839,7 @@ function Init { #${REPLICA[7]} contains full initiator last action file path #${REPLICA[8]} contains full target last action file path #${REPLICA[9]} contains full resume count file path + # Local variables used for state filenames local lock_filename="lock" local state_dir="state" @@ -2859,7 +2854,7 @@ function Init { local dry_suffix= fi - #TODO: integrate this into ${REPLICA[x]} + #TODO(low): integrate this into ${REPLICA[x]} TREE_CURRENT_FILENAME="-tree-current-$INSTANCE_ID$dry_suffix" TREE_AFTER_FILENAME="-tree-after-$INSTANCE_ID$dry_suffix" TREE_AFTER_FILENAME_NO_SUFFIX="-tree-after-$INSTANCE_ID" @@ -2932,22 +2927,6 @@ function Init { 'replica-tree-after' 'synced' ) - - ## Sync function actions (0-10) - #SYNC_ACTION=( - #'initiator-replica-tree' - #'target-replica-tree' - #'initiator-deleted-list' - #'target-deleted-list' - #'sync-attrs' - #'update-initiator-replica' - #'update-target-replica' - #'delete-propagation-target' - #'delete-propagation-initiator' - #'initiator-replica-tree-after' - #'target-replica-tree-after' - #'sync.success' - #) } function Main { @@ -3024,7 +3003,6 @@ function SyncOnChanges { Logger "#### Monitoring now." "NOTICE" inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & - #OSYNC_SUB_PID=$! Not used anymore with killchilds func wait $! retval=$? if [ $retval == 0 ]; then diff --git a/install.sh b/install.sh index 00bbe8b..9719734 100755 --- a/install.sh +++ b/install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash PROGRAM=osync -PROGRAM_VERSION=1.2-dev-parallel-unstable +PROGRAM_VERSION=1.2-dev-parallel PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SCRIPT_BUILD=2016052601 diff --git a/osync.sh b/osync.sh index 4766a00..25790fb 100755 --- a/osync.sh +++ b/osync.sh @@ -1,24 +1,21 @@ #!/usr/bin/env bash -#TODO: handle conflict prevalance +#TODO(critical): handle conflict prevalance, especially in sync_attrs function PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-dev-parallel-unstable -PROGRAM_BUILD=2016080901 +PROGRAM_VERSION=1.2-dev-parallel +PROGRAM_BUILD=2016081301 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016080903 +## FUNC_BUILD=2016081201 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr -#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode -#TODO: set _LOGGER_STDERR in other apps - ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" @@ -1218,9 +1215,11 @@ function InitRemoteOSSettings { #TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - #if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - # RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" - #fi + if [ "$PRESERVE_EXECUTABILITY" != "no" ];then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" + fi + fi if [ "$REMOTE_OS" == "msys" ]; then REMOTE_FIND_CMD=$(dirname $BASH)/find @@ -1352,7 +1351,7 @@ function CheckCurrentConfig { eval "$test" done - #TODO-v2.1: Add runtime variable tests (RSYNC_ARGS etc) + #TODO(low): Add runtime variable tests (RSYNC_ARGS etc) } ###### Osync specific functions (non shared) @@ -1412,7 +1411,7 @@ function CheckReplicaPaths { local pids - #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO: investigate realpath & readlink issues on MSYS and busybox here + #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO(verylow): investigate realpath & readlink issues on MSYS and busybox here #TARGET_SYNC_DIR_CANN=$(realpath "${TARGET[1]}) #if [ "$REMOTE_OPERATION" != "yes" ]; then @@ -2027,16 +2026,14 @@ function _delete_local { local deleted_failed_list_file="${4}" # file containing files that could not be deleted on last run, will be prefixed with replica type local parentdir - - ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted local previous_file="" - #TODO: fix and add same to remote - if [ -d "$replica_dir$deletion_dir" ]; then + if [ -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN -ne 1 ]; then $COMMAND_SUDO mkdir -p "$replica_dir$deletion_dir" fi while read -r files; do + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [ $_VERBOSE -eq 1 ]; then Logger "Soft deleting $replica_dir$files" "NOTICE" @@ -2104,13 +2101,13 @@ function _delete_remote { exit 1 fi -#TODO: check output file without & and sleep 5 +#TODO(critical): check output file without & and sleep 5 $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 ## The following lines are executed remotely function _logger { local value="${1}" # What to log - echo -e "$value" >&2 #TODO logfile output missing + echo -e "$value" >&2 #TODO(high): logfile output missing if [ $_SILENT -eq 0 ]; then echo -e "$value" @@ -2147,10 +2144,14 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= > "$FAILED_DELETE_LIST" parentdir= - - ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted previous_file="" + + if [ -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN -ne 1 ]; then + $COMMAND_SUDO mkdir -p "$replica_dir$deletion_dir" + fi + while read -r files; do + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [ $_VERBOSE -eq 1 ]; then Logger "Soft deleting $REPLICA_DIR$files" "NOTICE" @@ -2224,11 +2225,7 @@ function deletion_propagation { replica_dir="${INITIATOR[1]}" delete_dir="${INITIATOR[5]}" - - #TODO: parallelize this ! - - _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" retval=$? if [ $retval != 0 ]; then Logger "Deletion on replica $replica_type failed." "CRITICAL" @@ -2239,11 +2236,10 @@ function deletion_propagation { delete_dir="${TARGET[5]}" if [ "$REMOTE_OPERATION" == "yes" ]; then - _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & + _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" else - _delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & + _delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false retval=$? if [ $retval == 0 ]; then if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE -eq 1 ]; then @@ -2260,15 +2256,14 @@ function deletion_propagation { fi } -###### Sync function in 6 steps (functions above) +###### Sync function in 6 steps ###### -###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S) -###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S) -###### Steps 3a & 3b can be skipped depending on $RSYNC_ATTR_ARGS is empty or not -###### Step 3a: Update initiator and target file attributes only -###### Step 3b: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence) -###### Step 4: Deleted file propagation to initiator and target replicas (Steps 4M and 4S) -###### Step 5: Create after run tree list for initiator and target replicas (Steps 5M and 5S) +###### Step 0a & 0b: Create current file list of replicas +###### Step 1a & 1b: Create deleted file list of replicas +###### Step 3: Update file attributes +###### Step 3a & 3b: Update replicas +###### Step 4a & 4b: Propagate deletions on replicas +###### Step 5a & 5b: Create after run file list of replicas function Sync { @@ -2319,7 +2314,6 @@ function Sync { fi else Logger "Will not resume aborted execution. Too many resume tries [$resumeCount]." "WARN" - #echo "noresume" > "${INITIATOR[7]}" echo "0" > "${INITIATOR[9]}" resumeInitiator="none" resumeTarget="none" @@ -2537,14 +2531,14 @@ function _SoftDeleteLocal { if [ $_VERBOSE -eq 1 ]; then # Cannot launch log function from xargs, ugly hack - $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" - $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" + $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi if [ $_DRYRUN -ne 1 ]; then - $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 + $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 else Dummy fi @@ -2578,14 +2572,14 @@ function _SoftDeleteRemote { if [ $_VERBOSE -eq 1 ]; then # Cannot launch log function from xargs, ugly hack - cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi if [ $_DRYRUN -ne 1 ]; then - cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" @@ -2704,6 +2698,7 @@ function Init { #${REPLICA[7]} contains full initiator last action file path #${REPLICA[8]} contains full target last action file path #${REPLICA[9]} contains full resume count file path + # Local variables used for state filenames local lock_filename="lock" local state_dir="state" @@ -2718,7 +2713,7 @@ function Init { local dry_suffix= fi - #TODO: integrate this into ${REPLICA[x]} + #TODO(low): integrate this into ${REPLICA[x]} TREE_CURRENT_FILENAME="-tree-current-$INSTANCE_ID$dry_suffix" TREE_AFTER_FILENAME="-tree-after-$INSTANCE_ID$dry_suffix" TREE_AFTER_FILENAME_NO_SUFFIX="-tree-after-$INSTANCE_ID" @@ -2791,22 +2786,6 @@ function Init { 'replica-tree-after' 'synced' ) - - ## Sync function actions (0-10) - #SYNC_ACTION=( - #'initiator-replica-tree' - #'target-replica-tree' - #'initiator-deleted-list' - #'target-deleted-list' - #'sync-attrs' - #'update-initiator-replica' - #'update-target-replica' - #'delete-propagation-target' - #'delete-propagation-initiator' - #'initiator-replica-tree-after' - #'target-replica-tree-after' - #'sync.success' - #) } function Main { @@ -2880,7 +2859,6 @@ function SyncOnChanges { Logger "#### Monitoring now." "NOTICE" inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & - #OSYNC_SUB_PID=$! Not used anymore with killchilds func wait $! retval=$? if [ $retval == 0 ]; then