diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index bae6ba2..4560712 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.1-dev -PROGRAM_BUILD=2016021704 +PROGRAM_BUILD=2016021802 IS_STABLE=no FUNC_BUILD=2016021604 @@ -1064,10 +1064,12 @@ function _CreateStateDirsRemote { local replica_state_dir="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1119,9 +1121,11 @@ function _CheckReplicaPathsRemote { local replica_path="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1131,7 +1135,7 @@ function _CheckReplicaPathsRemote { exit 1 fi - local cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' + cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1180,10 +1184,12 @@ function _CheckDiskSpaceRemote { Logger "Checking minimum disk space on target [$replica_path]." "NOTICE" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1212,12 +1218,13 @@ function CheckDiskSpace { function RsyncPatternsAdd { local pattern="${1}" local pattern_type="${2}" # exclude or include - __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local rest= + # Disable globbing so wildcards from exclusions do not get expanded set -f - local rest="$pattern" + rest="$pattern" while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR @@ -1241,9 +1248,10 @@ function RsyncPatternsAdd { function RsyncPatternsFromAdd { local pattern_from="${1}" local pattern_type="${2}" - __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local pattern_from= + ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename $pattern_from)" == "$pattern_from" ]; then pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" @@ -1297,10 +1305,12 @@ function _WriteLockFilesRemote { local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' + cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1327,11 +1337,15 @@ function _CheckLocksLocal { local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local lockfile_content= + local lock_pid= + local lock_instance_id= + if [ -f "$lockfile" ]; then - local lockfile_content=$(cat $lockfile) + lockfile_content=$(cat $lockfile) Logger "Master lock pid present: $lockfile_content" "DEBUG" - local lock_pid=${lockfile_content%@*} - local lock_instance_id=${lockfile_content#*@} + lock_pid=${lockfile_content%@*} + lock_instance_id=${lockfile_content#*@} ps -p$lock_pid > /dev/null 2>&1 if [ $? != 0 ]; then Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" @@ -1346,10 +1360,14 @@ function _CheckLocksRemote { #TODO: Rewrite this a bit more beautiful local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + local lock_pid= + local lock_instance_id= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'"' + cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'"' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1362,8 +1380,8 @@ function _CheckLocksRemote { #TODO: Rewrite this a bit more beautiful fi fi - local lock_pid=${lockfile_content%@*} - local lock_instance_id=${lockfile_content#*@} + lock_pid=${lockfile_content%@*} + lock_instance_id=${lockfile_content#*@} if [ "$lock_pid" != "" ] && [ "$lock_instance_id" != "" ]; then Logger "Remote lock is: $lock_pid@$lock_instance_id" "DEBUG" @@ -1429,10 +1447,12 @@ function _UnlockReplicasRemote { local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1470,17 +1490,21 @@ function tree_list { local replica_path="${1}" # path to the replica for which a tree needs to be constructed local replica_type="${2}" # replica type: initiator, target local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type) + + local escaped_replica_path= + local rsync_cmd= + __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - local escaped_replica_path=$(EscapeSpaces "$replica_path") + escaped_replica_path=$(EscapeSpaces "$replica_path") Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE" if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" else - local rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" fi Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" ## Redirect commands stderr here to get rsync stderr output in logfile @@ -1506,6 +1530,8 @@ function delete_list { local deleted_failed_list_file="${5}" # file containing files that could not be deleted on last run, will be prefixed with replica type __CheckArguments 5 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + # TODO: Check why external filenames are used (see _DRYRUN option because of NOSUFFIX) Logger "Creating $replica_type replica deleted file list." "NOTICE" @@ -1541,7 +1567,6 @@ function _get_file_ctime_mtime_local { local file_list="${3}" # Contains list of files to get time attrs __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - # TODO: make this more beautiful (include path in stat command ?) cat "$file_list" | xargs -I {} stat -c '%n|%Z|%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID" } @@ -1551,17 +1576,30 @@ function _get_file_ctime_mtime_remote { local file_list="${3}" __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - #cmd='cat "'$file_list'" | '$SSH_CMD' cd "'$replica_path'"; xargs -I {} stat -c \'%n|%Z|%Y\' "{}" | sort > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID'"' + local cmd= + + cmd='cat "'$file_list'" | '$SSH_CMD' xargs -I "'$replica_path'"{} stat -c "%n|%Z|%Y" "{}" | sort > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID'"' Logger "CMD: $cmd" "DEBUG" eval "$cmd" + WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME + if [ $? != 0 ]; then + Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL" + if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID)" "NOTICE" + fi + exit 1 + fi } +# rsync does sync with mtime, but file attribute modifications only change ctime. +# Hence, detect newer ctime on the replica that gets updated first with CONFLICT_PREVALANCE and update all newer file attributes on this replica before real update function sync_attrs { - local initiator_replica="${1}" # Contains #TODO: Write ACL update function, check other ctime related attributes (xattr ?) + local initiator_replica="${1}" local target_replica="${2}" __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG local rsync_cmd= + local retval= Logger "Getting file attributes." "NOTICE" @@ -1581,15 +1619,15 @@ function sync_attrs { fi if [ $retval != 0 ] && [ $retval != 24 ]; then - Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL" + Logger "Getting file list for attribute sync failed [$retval]. Stopping execution." "CRITICAL" if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" fi exit $retval else - cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | ( sed -e 's/^[^ ]* //' || :) > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" + cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" if [ $? != 0 ]; then - Logger "Cannot prepare file attribute list." "CRITICAL" + Logger "Cannot prepare file list for attribute sync." "CRITICAL" exit 1 fi fi @@ -1601,8 +1639,22 @@ function sync_attrs { _get_file_ctime_mtime_remote "${TARGET[1]}" "${TARGET[0]}" "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" fi - #WIP - join -j 1 -t ';' -o 1.1,1.2,2.2 init targ | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + # If target gets updated first, then sync_attr must update initiator's attrs first + # Also, change replica paths of the two file lists so rsync will know what to sync + if [ "$CONFLICT_PREVALANCE" == "${INITIATOR[0]}" ]; then + source=targ + dest=ini + sed -i "s;^${INITIATOR[1]};${TARGET[1]};g" "$RUN_DIR/$PROGRAM.syncattr.$SCRIPT_PID" + join -j 1 -t ';' -o 1.1,1.2,2.2 targ init | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + + else + + source=ini + dest=tar + sed -i "s;^${TARGET[1]};${INITIATOR[1]};g" "$RUN_DIR/$PROGRAM.syncattr.$SCRIPT_PID" + join -j 1 -t ';' -o 1.1,1.2,2.2 init targ | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + fi + if [ "$REMOTE_OPERATION" != "yes" ]; then rsync_cmd="" else @@ -1617,13 +1669,13 @@ function sync_attrs { fi if [ $retval != 0 ] && [ $retval != 24 ]; then - Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL" + Logger "Updating file attributes on $source [$retval]. Stopping execution." "CRITICAL" if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID)" "NOTICE" fi exit $retval else - Logger "Successfully updated file attributes on target replica." "NOTICE" + Logger "Successfully updated file attributes on $dest replica." "NOTICE" fi } @@ -1634,6 +1686,9 @@ function sync_update { local delete_list_filename="${3}" # Contains deleted list filename, will be prefixed with replica type __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local rsync_cmd= + local retval= + Logger "Updating $destination_replica replica." "NOTICE" if [ "$source_replica" == "${INITIATOR[0]}" ]; then local source_dir="${INITIATOR[1]}" @@ -1688,8 +1743,10 @@ function _delete_local { local deleted_failed_list_file="${4}" # file containing files that could not be deleted on last run, will be prefixed with replica type __CheckArguments 4 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local parentdir= + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted - previous_file="" + local previous_file="" OLD_IFS=$IFS IFS=$'\r\n' for files in $(cat "${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file") @@ -1782,7 +1839,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= local value="${1}" # What to log local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL - prefix="RTIME: $SECONDS - " + local prefix="RTIME: $SECONDS - " if [ "$level" == "CRITICAL" ]; then _logger "$prefix\e[41m$value\e[0m" @@ -1807,8 +1864,10 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= ## Empty earlier failed delete list > "$FAILED_DELETE_LIST" + local parentdir= + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted - previous_file="" + local previous_file="" OLD_IFS=$IFS IFS=$'\r\n' for files in $(cat "$FILE_LIST") @@ -1891,11 +1950,14 @@ function deletion_propagation { local deleted_failed_list_file="${3}" # file containing files that could not be deleted on last run, will be prefixed with replica type __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local replica_dir= + local delete_dir= + Logger "Propagating deletions to $replica_type replica." "NOTICE" if [ "$replica_type" == "{$INITIATOR[0]}" ]; then - local replica_dir="${INITIATOR[1]}${INITIATOR[3]}/" - local delete_dir="${INITIATOR[5]}" + replica_dir="${INITIATOR[1]}${INITIATOR[3]}/" + delete_dir="${INITIATOR[5]}" _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" & WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME @@ -1905,8 +1967,8 @@ function deletion_propagation { exit 1 fi else - local replica_dir="${TARGET[1]}${TARGET[3]}/" - local delete_dir="${TARGET[5]}" + replica_dir="${TARGET[1]}${TARGET[3]}/" + delete_dir="${TARGET[5]}" if [ "$REMOTE_OPERATION" == "yes" ]; then _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & @@ -2025,7 +2087,7 @@ function Sync { resume_sync="resumed" fi if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ]; then - if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then + if [ "$CONFLICT_PREVALANCE" != "${INITIATOR[0]}" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then sync_update ${TARGET[0]} ${INITIATOR[0]} "$DELETED_LIST_FILENAME" if [ $? == 0 ]; then @@ -2116,6 +2178,8 @@ function _SoftDeleteLocal { local change_time="${3}" __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local retval= + if [ -d "$replica_deletion_path" ]; then if [ $_DRYRUN -eq 1 ]; then Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE" @@ -2153,6 +2217,8 @@ function _SoftDeleteRemote { local change_time="${3}" __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local retval= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -2230,6 +2296,11 @@ function Init { trap TrapQuit SIGTERM EXIT SIGKILL SIGHUP SIGQUIT fi + local uri + local hosturiandpath + local hosturi + + ## Test if target dir is a ssh uri, and if yes, break it down it its values if [ "${TARGET_SYNC_DIR:0:6}" == "ssh://" ]; then REMOTE_OPERATION="yes" @@ -2248,24 +2319,28 @@ function Init { fi # remove everything before '@' - _hosturiandpath=${uri#*@} + hosturiandpath=${uri#*@} # remove everything after first '/' - _hosturi=${_hosturiandpath%%/*} - if [[ "$_hosturi" == *":"* ]]; then - REMOTE_PORT=${_hosturi##*:} + hosturi=${hosturiandpath%%/*} + if [[ "$hosturi" == *":"* ]]; then + REMOTE_PORT=${hosturi##*:} else REMOTE_PORT=22 fi - REMOTE_HOST=${_hosturi%%:*} + REMOTE_HOST=${hosturi%%:*} # remove everything before first '/' - TARGET_SYNC_DIR=${_hosturiandpath#*/} + TARGET_SYNC_DIR=${hosturiandpath#*/} fi ## Make sure there is only one trailing slash on path INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR%/}/" TARGET_SYNC_DIR="${TARGET_SYNC_DIR%/}/" + if [ $_DRYRUN -eq 1 ]; then + dry_suffix="-dry" + fi + ## Replica format ## Why the f*** does bash not have simple objects ? @@ -2323,7 +2398,7 @@ function Init { SYNC_OPTS=$SYNC_OPTS"i" fi - if [ $stats -eq 1 ]; then + if [ $STATS -eq 1 ]; then SYNC_OPTS=$SYNC_OPTS" --stats" fi @@ -2332,11 +2407,6 @@ function Init { RsyncPatterns fi - ## Filenames for state files - if [ $_DRYRUN -eq 1 ]; then - dry_suffix="-dry" - fi - ## Conflict options if [ "$CONFLICT_BACKUP" != "no" ]; then INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[1]}${INITIATOR[4]}\"" @@ -2422,6 +2492,9 @@ function Usage { function SyncOnChanges { __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + local retval= + if ! type inotifywait > /dev/null 2>&1 ; then Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL" exit 1 @@ -2446,8 +2519,8 @@ function SyncOnChanges { Logger "#### Monitoring now." "NOTICE" inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & - OSYNC_SUB_PID=$! - wait $OSYNC_SUB_PID + #OSYNC_SUB_PID=$! Not used anymore with killchilds func + wait $! retval=$? if [ $retval == 0 ]; then Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE" @@ -2462,7 +2535,7 @@ function SyncOnChanges { } -stats=0 +STATS=0 PARTIAL=0 FORCE_UNLOCK=0 no_maxtime=0 @@ -2612,6 +2685,7 @@ opts="${opts# *}" GetLocalOS InitLocalOSSettings + CheckEnvironment PreInit Init PostInit diff --git a/dev/n_osync.sh b/dev/n_osync.sh index f2a0945..6efa158 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.1-dev -PROGRAM_BUILD=2016021704 +PROGRAM_BUILD=2016021802 IS_STABLE=no source "./ofunctions.sh" @@ -149,10 +149,12 @@ function _CreateStateDirsRemote { local replica_state_dir="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -204,9 +206,11 @@ function _CheckReplicaPathsRemote { local replica_path="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -216,7 +220,7 @@ function _CheckReplicaPathsRemote { exit 1 fi - local cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' + cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -265,10 +269,12 @@ function _CheckDiskSpaceRemote { Logger "Checking minimum disk space on target [$replica_path]." "NOTICE" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -297,12 +303,13 @@ function CheckDiskSpace { function RsyncPatternsAdd { local pattern="${1}" local pattern_type="${2}" # exclude or include - __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local rest= + # Disable globbing so wildcards from exclusions do not get expanded set -f - local rest="$pattern" + rest="$pattern" while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR @@ -326,8 +333,9 @@ function RsyncPatternsAdd { function RsyncPatternsFromAdd { local pattern_from="${1}" local pattern_type="${2}" + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local pattern_from= ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename $pattern_from)" == "$pattern_from" ]; then @@ -382,10 +390,12 @@ function _WriteLockFilesRemote { local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' + cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -412,11 +422,15 @@ function _CheckLocksLocal { local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local lockfile_content= + local lock_pid= + local lock_instance_id= + if [ -f "$lockfile" ]; then - local lockfile_content=$(cat $lockfile) + lockfile_content=$(cat $lockfile) Logger "Master lock pid present: $lockfile_content" "DEBUG" - local lock_pid=${lockfile_content%@*} - local lock_instance_id=${lockfile_content#*@} + lock_pid=${lockfile_content%@*} + lock_instance_id=${lockfile_content#*@} ps -p$lock_pid > /dev/null 2>&1 if [ $? != 0 ]; then Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" @@ -431,10 +445,14 @@ function _CheckLocksRemote { #TODO: Rewrite this a bit more beautiful local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + local lock_pid= + local lock_instance_id= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'"' + cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'"' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -447,8 +465,8 @@ function _CheckLocksRemote { #TODO: Rewrite this a bit more beautiful fi fi - local lock_pid=${lockfile_content%@*} - local lock_instance_id=${lockfile_content#*@} + lock_pid=${lockfile_content%@*} + lock_instance_id=${lockfile_content#*@} if [ "$lock_pid" != "" ] && [ "$lock_instance_id" != "" ]; then Logger "Remote lock is: $lock_pid@$lock_instance_id" "DEBUG" @@ -514,10 +532,12 @@ function _UnlockReplicasRemote { local lockfile="${1}" __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -555,17 +575,21 @@ function tree_list { local replica_path="${1}" # path to the replica for which a tree needs to be constructed local replica_type="${2}" # replica type: initiator, target local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type) + + local escaped_replica_path= + local rsync_cmd= + __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - local escaped_replica_path=$(EscapeSpaces "$replica_path") + escaped_replica_path=$(EscapeSpaces "$replica_path") Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE" if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" else - local rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" fi Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" ## Redirect commands stderr here to get rsync stderr output in logfile @@ -591,6 +615,8 @@ function delete_list { local deleted_failed_list_file="${5}" # file containing files that could not be deleted on last run, will be prefixed with replica type __CheckArguments 5 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + # TODO: Check why external filenames are used (see _DRYRUN option because of NOSUFFIX) Logger "Creating $replica_type replica deleted file list." "NOTICE" @@ -626,7 +652,6 @@ function _get_file_ctime_mtime_local { local file_list="${3}" # Contains list of files to get time attrs __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - # TODO: make this more beautiful (include path in stat command ?) cat "$file_list" | xargs -I {} stat -c '%n|%Z|%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID" } @@ -636,17 +661,30 @@ function _get_file_ctime_mtime_remote { local file_list="${3}" __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - #cmd='cat "'$file_list'" | '$SSH_CMD' cd "'$replica_path'"; xargs -I {} stat -c \'%n|%Z|%Y\' "{}" | sort > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID'"' + local cmd= + + cmd='cat "'$file_list'" | '$SSH_CMD' xargs -I "'$replica_path'"{} stat -c "%n|%Z|%Y" "{}" | sort > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID'"' Logger "CMD: $cmd" "DEBUG" eval "$cmd" + WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME + if [ $? != 0 ]; then + Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL" + if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID)" "NOTICE" + fi + exit 1 + fi } +# rsync does sync with mtime, but file attribute modifications only change ctime. +# Hence, detect newer ctime on the replica that gets updated first with CONFLICT_PREVALANCE and update all newer file attributes on this replica before real update function sync_attrs { - local initiator_replica="${1}" # Contains #TODO: Write ACL update function, check other ctime related attributes (xattr ?) + local initiator_replica="${1}" local target_replica="${2}" __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG local rsync_cmd= + local retval= Logger "Getting file attributes." "NOTICE" @@ -666,15 +704,15 @@ function sync_attrs { fi if [ $retval != 0 ] && [ $retval != 24 ]; then - Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL" + Logger "Getting file list for attribute sync failed [$retval]. Stopping execution." "CRITICAL" if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" fi exit $retval else - cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | ( sed -e 's/^[^ ]* //' || :) > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" + cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" if [ $? != 0 ]; then - Logger "Cannot prepare file attribute list." "CRITICAL" + Logger "Cannot prepare file list for attribute sync." "CRITICAL" exit 1 fi fi @@ -686,8 +724,22 @@ function sync_attrs { _get_file_ctime_mtime_remote "${TARGET[1]}" "${TARGET[0]}" "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" fi - #WIP - join -j 1 -t ';' -o 1.1,1.2,2.2 init targ | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + # If target gets updated first, then sync_attr must update initiator's attrs first + # Also, change replica paths of the two file lists so rsync will know what to sync + if [ "$CONFLICT_PREVALANCE" == "${INITIATOR[0]}" ]; then + source=targ + dest=ini + sed -i "s;^${INITIATOR[1]};${TARGET[1]};g" "$RUN_DIR/$PROGRAM.syncattr.$SCRIPT_PID" + join -j 1 -t ';' -o 1.1,1.2,2.2 targ init | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + + else + + source=ini + dest=tar + sed -i "s;^${TARGET[1]};${INITIATOR[1]};g" "$RUN_DIR/$PROGRAM.syncattr.$SCRIPT_PID" + join -j 1 -t ';' -o 1.1,1.2,2.2 init targ | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + fi + if [ "$REMOTE_OPERATION" != "yes" ]; then rsync_cmd="" else @@ -702,13 +754,13 @@ function sync_attrs { fi if [ $retval != 0 ] && [ $retval != 24 ]; then - Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL" + Logger "Updating file attributes on $source [$retval]. Stopping execution." "CRITICAL" if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID)" "NOTICE" fi exit $retval else - Logger "Successfully updated file attributes on target replica." "NOTICE" + Logger "Successfully updated file attributes on $dest replica." "NOTICE" fi } @@ -719,6 +771,9 @@ function sync_update { local delete_list_filename="${3}" # Contains deleted list filename, will be prefixed with replica type __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local rsync_cmd= + local retval= + Logger "Updating $destination_replica replica." "NOTICE" if [ "$source_replica" == "${INITIATOR[0]}" ]; then local source_dir="${INITIATOR[1]}" @@ -773,8 +828,10 @@ function _delete_local { local deleted_failed_list_file="${4}" # file containing files that could not be deleted on last run, will be prefixed with replica type __CheckArguments 4 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local parentdir= + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted - previous_file="" + local previous_file="" OLD_IFS=$IFS IFS=$'\r\n' for files in $(cat "${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file") @@ -867,7 +924,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= local value="${1}" # What to log local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL - prefix="RTIME: $SECONDS - " + local prefix="RTIME: $SECONDS - " if [ "$level" == "CRITICAL" ]; then _logger "$prefix\e[41m$value\e[0m" @@ -892,8 +949,10 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= ## Empty earlier failed delete list > "$FAILED_DELETE_LIST" + local parentdir= + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted - previous_file="" + local previous_file="" OLD_IFS=$IFS IFS=$'\r\n' for files in $(cat "$FILE_LIST") @@ -976,11 +1035,14 @@ function deletion_propagation { local deleted_failed_list_file="${3}" # file containing files that could not be deleted on last run, will be prefixed with replica type __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local replica_dir= + local delete_dir= + Logger "Propagating deletions to $replica_type replica." "NOTICE" if [ "$replica_type" == "{$INITIATOR[0]}" ]; then - local replica_dir="${INITIATOR[1]}${INITIATOR[3]}/" - local delete_dir="${INITIATOR[5]}" + replica_dir="${INITIATOR[1]}${INITIATOR[3]}/" + delete_dir="${INITIATOR[5]}" _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" & WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME @@ -990,8 +1052,8 @@ function deletion_propagation { exit 1 fi else - local replica_dir="${TARGET[1]}${TARGET[3]}/" - local delete_dir="${TARGET[5]}" + replica_dir="${TARGET[1]}${TARGET[3]}/" + delete_dir="${TARGET[5]}" if [ "$REMOTE_OPERATION" == "yes" ]; then _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & @@ -1110,7 +1172,7 @@ function Sync { resume_sync="resumed" fi if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ]; then - if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then + if [ "$CONFLICT_PREVALANCE" != "${INITIATOR[0]}" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then sync_update ${TARGET[0]} ${INITIATOR[0]} "$DELETED_LIST_FILENAME" if [ $? == 0 ]; then @@ -1201,6 +1263,8 @@ function _SoftDeleteLocal { local change_time="${3}" __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local retval= + if [ -d "$replica_deletion_path" ]; then if [ $_DRYRUN -eq 1 ]; then Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE" @@ -1238,6 +1302,8 @@ function _SoftDeleteRemote { local change_time="${3}" __CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local retval= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -1315,6 +1381,11 @@ function Init { trap TrapQuit SIGTERM EXIT SIGKILL SIGHUP SIGQUIT fi + local uri + local hosturiandpath + local hosturi + + ## Test if target dir is a ssh uri, and if yes, break it down it its values if [ "${TARGET_SYNC_DIR:0:6}" == "ssh://" ]; then REMOTE_OPERATION="yes" @@ -1333,24 +1404,28 @@ function Init { fi # remove everything before '@' - _hosturiandpath=${uri#*@} + hosturiandpath=${uri#*@} # remove everything after first '/' - _hosturi=${_hosturiandpath%%/*} - if [[ "$_hosturi" == *":"* ]]; then - REMOTE_PORT=${_hosturi##*:} + hosturi=${hosturiandpath%%/*} + if [[ "$hosturi" == *":"* ]]; then + REMOTE_PORT=${hosturi##*:} else REMOTE_PORT=22 fi - REMOTE_HOST=${_hosturi%%:*} + REMOTE_HOST=${hosturi%%:*} # remove everything before first '/' - TARGET_SYNC_DIR=${_hosturiandpath#*/} + TARGET_SYNC_DIR=${hosturiandpath#*/} fi ## Make sure there is only one trailing slash on path INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR%/}/" TARGET_SYNC_DIR="${TARGET_SYNC_DIR%/}/" + if [ $_DRYRUN -eq 1 ]; then + dry_suffix="-dry" + fi + ## Replica format ## Why the f*** does bash not have simple objects ? @@ -1408,7 +1483,7 @@ function Init { SYNC_OPTS=$SYNC_OPTS"i" fi - if [ $stats -eq 1 ]; then + if [ $STATS -eq 1 ]; then SYNC_OPTS=$SYNC_OPTS" --stats" fi @@ -1417,11 +1492,6 @@ function Init { RsyncPatterns fi - ## Filenames for state files - if [ $_DRYRUN -eq 1 ]; then - dry_suffix="-dry" - fi - ## Conflict options if [ "$CONFLICT_BACKUP" != "no" ]; then INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[1]}${INITIATOR[4]}\"" @@ -1507,6 +1577,9 @@ function Usage { function SyncOnChanges { __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + local cmd= + local retval= + if ! type inotifywait > /dev/null 2>&1 ; then Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL" exit 1 @@ -1531,8 +1604,8 @@ function SyncOnChanges { Logger "#### Monitoring now." "NOTICE" inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & - OSYNC_SUB_PID=$! - wait $OSYNC_SUB_PID + #OSYNC_SUB_PID=$! Not used anymore with killchilds func + wait $! retval=$? if [ $retval == 0 ]; then Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE" @@ -1547,7 +1620,7 @@ function SyncOnChanges { } -stats=0 +STATS=0 PARTIAL=0 FORCE_UNLOCK=0 no_maxtime=0 @@ -1697,6 +1770,7 @@ opts="${opts# *}" GetLocalOS InitLocalOSSettings + CheckEnvironment PreInit Init PostInit diff --git a/osync.sh b/osync.sh index 3836103..0608cbc 100755 --- a/osync.sh +++ b/osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.1-dev -PROGRAM_BUILD=2016021704 +PROGRAM_BUILD=2016021802 IS_STABLE=no FUNC_BUILD=2016021604 @@ -959,10 +959,12 @@ function _CreateStateDirsLocal { function _CreateStateDirsRemote { local replica_state_dir="${1}" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1011,9 +1013,11 @@ function _CheckReplicaPathsLocal { function _CheckReplicaPathsRemote { local replica_path="${1}" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1023,7 +1027,7 @@ function _CheckReplicaPathsRemote { exit 1 fi - local cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' + cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1069,10 +1073,12 @@ function _CheckDiskSpaceRemote { Logger "Checking minimum disk space on target [$replica_path]." "NOTICE" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1101,10 +1107,11 @@ function RsyncPatternsAdd { local pattern="${1}" local pattern_type="${2}" # exclude or include + local rest= # Disable globbing so wildcards from exclusions do not get expanded set -f - local rest="$pattern" + rest="$pattern" while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR @@ -1129,6 +1136,7 @@ function RsyncPatternsFromAdd { local pattern_from="${1}" local pattern_type="${2}" + local pattern_from= ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename $pattern_from)" == "$pattern_from" ]; then @@ -1180,10 +1188,12 @@ function _WriteLockFilesLocal { function _WriteLockFilesRemote { local lockfile="${1}" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' + cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1208,11 +1218,15 @@ function WriteLockFiles { function _CheckLocksLocal { local lockfile="${1}" + local lockfile_content= + local lock_pid= + local lock_instance_id= + if [ -f "$lockfile" ]; then - local lockfile_content=$(cat $lockfile) + lockfile_content=$(cat $lockfile) Logger "Master lock pid present: $lockfile_content" "DEBUG" - local lock_pid=${lockfile_content%@*} - local lock_instance_id=${lockfile_content#*@} + lock_pid=${lockfile_content%@*} + lock_instance_id=${lockfile_content#*@} ps -p$lock_pid > /dev/null 2>&1 if [ $? != 0 ]; then Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" @@ -1226,10 +1240,14 @@ function _CheckLocksLocal { function _CheckLocksRemote { #TODO: Rewrite this a bit more beautiful local lockfile="${1}" + local cmd= + local lock_pid= + local lock_instance_id= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'"' + cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'"' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1242,8 +1260,8 @@ function _CheckLocksRemote { #TODO: Rewrite this a bit more beautiful fi fi - local lock_pid=${lockfile_content%@*} - local lock_instance_id=${lockfile_content#*@} + lock_pid=${lockfile_content%@*} + lock_instance_id=${lockfile_content#*@} if [ "$lock_pid" != "" ] && [ "$lock_instance_id" != "" ]; then Logger "Remote lock is: $lock_pid@$lock_instance_id" "DEBUG" @@ -1306,10 +1324,12 @@ function _UnlockReplicasLocal { function _UnlockReplicasRemote { local lockfile="${1}" + local cmd= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 720 1800 $FUNCNAME @@ -1347,15 +1367,19 @@ function tree_list { local replica_type="${2}" # replica type: initiator, target local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type) - local escaped_replica_path=$(EscapeSpaces "$replica_path") + local escaped_replica_path= + local rsync_cmd= + + + escaped_replica_path=$(EscapeSpaces "$replica_path") Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE" if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - local rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" else - local rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" fi Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" ## Redirect commands stderr here to get rsync stderr output in logfile @@ -1380,6 +1404,8 @@ function delete_list { local deleted_list_file="${4}" # file containing deleted file list, will be prefixed with replica type local deleted_failed_list_file="${5}" # file containing files that could not be deleted on last run, will be prefixed with replica type + local cmd= + # TODO: Check why external filenames are used (see _DRYRUN option because of NOSUFFIX) Logger "Creating $replica_type replica deleted file list." "NOTICE" @@ -1414,7 +1440,6 @@ function _get_file_ctime_mtime_local { local replica_type="${2}" # Initiator / Target local file_list="${3}" # Contains list of files to get time attrs - # TODO: make this more beautiful (include path in stat command ?) cat "$file_list" | xargs -I {} stat -c '%n|%Z|%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID" } @@ -1423,16 +1448,29 @@ function _get_file_ctime_mtime_remote { local replica_type="${2}" local file_list="${3}" - #cmd='cat "'$file_list'" | '$SSH_CMD' cd "'$replica_path'"; xargs -I {} stat -c \'%n|%Z|%Y\' "{}" | sort > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID'"' + local cmd= + + cmd='cat "'$file_list'" | '$SSH_CMD' xargs -I "'$replica_path'"{} stat -c "%n|%Z|%Y" "{}" | sort > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID'"' Logger "CMD: $cmd" "DEBUG" eval "$cmd" + WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME + if [ $? != 0 ]; then + Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL" + if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$replica_type.$SCRIPT_PID)" "NOTICE" + fi + exit 1 + fi } +# rsync does sync with mtime, but file attribute modifications only change ctime. +# Hence, detect newer ctime on the replica that gets updated first with CONFLICT_PREVALANCE and update all newer file attributes on this replica before real update function sync_attrs { - local initiator_replica="${1}" # Contains #TODO: Write ACL update function, check other ctime related attributes (xattr ?) + local initiator_replica="${1}" local target_replica="${2}" local rsync_cmd= + local retval= Logger "Getting file attributes." "NOTICE" @@ -1452,15 +1490,15 @@ function sync_attrs { fi if [ $retval != 0 ] && [ $retval != 24 ]; then - Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL" + Logger "Getting file list for attribute sync failed [$retval]. Stopping execution." "CRITICAL" if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" fi exit $retval else - cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | ( sed -e 's/^[^ ]* //' || :) > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" + cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" if [ $? != 0 ]; then - Logger "Cannot prepare file attribute list." "CRITICAL" + Logger "Cannot prepare file list for attribute sync." "CRITICAL" exit 1 fi fi @@ -1472,8 +1510,22 @@ function sync_attrs { _get_file_ctime_mtime_remote "${TARGET[1]}" "${TARGET[0]}" "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID" fi - #WIP - join -j 1 -t ';' -o 1.1,1.2,2.2 init targ | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + # If target gets updated first, then sync_attr must update initiator's attrs first + # Also, change replica paths of the two file lists so rsync will know what to sync + if [ "$CONFLICT_PREVALANCE" == "${INITIATOR[0]}" ]; then + source=targ + dest=ini + sed -i "s;^${INITIATOR[1]};${TARGET[1]};g" "$RUN_DIR/$PROGRAM.syncattr.$SCRIPT_PID" + join -j 1 -t ';' -o 1.1,1.2,2.2 targ init | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + + else + + source=ini + dest=tar + sed -i "s;^${TARGET[1]};${INITIATOR[1]};g" "$RUN_DIR/$PROGRAM.syncattr.$SCRIPT_PID" + join -j 1 -t ';' -o 1.1,1.2,2.2 init targ | awk -F';' '{if ($2 < $3) print $1}' > "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" + fi + if [ "$REMOTE_OPERATION" != "yes" ]; then rsync_cmd="" else @@ -1488,13 +1540,13 @@ function sync_attrs { fi if [ $retval != 0 ] && [ $retval != 24 ]; then - Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL" + Logger "Updating file attributes on $source [$retval]. Stopping execution." "CRITICAL" if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME-attrfiles.$SCRIPT_PID)" "NOTICE" fi exit $retval else - Logger "Successfully updated file attributes on target replica." "NOTICE" + Logger "Successfully updated file attributes on $dest replica." "NOTICE" fi } @@ -1504,6 +1556,9 @@ function sync_update { local destination_replica="${2}" # Contains replica type of destination: initiator, target local delete_list_filename="${3}" # Contains deleted list filename, will be prefixed with replica type + local rsync_cmd= + local retval= + Logger "Updating $destination_replica replica." "NOTICE" if [ "$source_replica" == "${INITIATOR[0]}" ]; then local source_dir="${INITIATOR[1]}" @@ -1557,8 +1612,10 @@ function _delete_local { local deletion_dir="${3}" # deletion dir in format .[workdir]/deleted local deleted_failed_list_file="${4}" # file containing files that could not be deleted on last run, will be prefixed with replica type + local parentdir= + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted - previous_file="" + local previous_file="" OLD_IFS=$IFS IFS=$'\r\n' for files in $(cat "${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file") @@ -1650,7 +1707,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= local value="${1}" # What to log local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL - prefix="RTIME: $SECONDS - " + local prefix="RTIME: $SECONDS - " if [ "$level" == "CRITICAL" ]; then _logger "$prefix\e[41m$value\e[0m" @@ -1675,8 +1732,10 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG= ## Empty earlier failed delete list > "$FAILED_DELETE_LIST" + local parentdir= + ## On every run, check wheter the next item is already deleted because it is included in a directory already deleted - previous_file="" + local previous_file="" OLD_IFS=$IFS IFS=$'\r\n' for files in $(cat "$FILE_LIST") @@ -1758,11 +1817,14 @@ function deletion_propagation { local deleted_list_file="${2}" # file containing deleted file list, will be prefixed with replica type local deleted_failed_list_file="${3}" # file containing files that could not be deleted on last run, will be prefixed with replica type + local replica_dir= + local delete_dir= + Logger "Propagating deletions to $replica_type replica." "NOTICE" if [ "$replica_type" == "{$INITIATOR[0]}" ]; then - local replica_dir="${INITIATOR[1]}${INITIATOR[3]}/" - local delete_dir="${INITIATOR[5]}" + replica_dir="${INITIATOR[1]}${INITIATOR[3]}/" + delete_dir="${INITIATOR[5]}" _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" & WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME @@ -1772,8 +1834,8 @@ function deletion_propagation { exit 1 fi else - local replica_dir="${TARGET[1]}${TARGET[3]}/" - local delete_dir="${TARGET[5]}" + replica_dir="${TARGET[1]}${TARGET[3]}/" + delete_dir="${TARGET[5]}" if [ "$REMOTE_OPERATION" == "yes" ]; then _delete_remote "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & @@ -1891,7 +1953,7 @@ function Sync { resume_sync="resumed" fi if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ]; then - if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then + if [ "$CONFLICT_PREVALANCE" != "${INITIATOR[0]}" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then sync_update ${TARGET[0]} ${INITIATOR[0]} "$DELETED_LIST_FILENAME" if [ $? == 0 ]; then @@ -1981,6 +2043,8 @@ function _SoftDeleteLocal { local replica_deletion_path="${2}" # Contains the full path to softdelete / backup directory without ending slash local change_time="${3}" + local retval= + if [ -d "$replica_deletion_path" ]; then if [ $_DRYRUN -eq 1 ]; then Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE" @@ -2017,6 +2081,8 @@ function _SoftDeleteRemote { local replica_deletion_path="${2}" # Contains the full path to softdelete / backup directory without ending slash local change_time="${3}" + local retval= + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -2092,6 +2158,11 @@ function Init { trap TrapQuit SIGTERM EXIT SIGKILL SIGHUP SIGQUIT fi + local uri + local hosturiandpath + local hosturi + + ## Test if target dir is a ssh uri, and if yes, break it down it its values if [ "${TARGET_SYNC_DIR:0:6}" == "ssh://" ]; then REMOTE_OPERATION="yes" @@ -2110,24 +2181,28 @@ function Init { fi # remove everything before '@' - _hosturiandpath=${uri#*@} + hosturiandpath=${uri#*@} # remove everything after first '/' - _hosturi=${_hosturiandpath%%/*} - if [[ "$_hosturi" == *":"* ]]; then - REMOTE_PORT=${_hosturi##*:} + hosturi=${hosturiandpath%%/*} + if [[ "$hosturi" == *":"* ]]; then + REMOTE_PORT=${hosturi##*:} else REMOTE_PORT=22 fi - REMOTE_HOST=${_hosturi%%:*} + REMOTE_HOST=${hosturi%%:*} # remove everything before first '/' - TARGET_SYNC_DIR=${_hosturiandpath#*/} + TARGET_SYNC_DIR=${hosturiandpath#*/} fi ## Make sure there is only one trailing slash on path INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR%/}/" TARGET_SYNC_DIR="${TARGET_SYNC_DIR%/}/" + if [ $_DRYRUN -eq 1 ]; then + dry_suffix="-dry" + fi + ## Replica format ## Why the f*** does bash not have simple objects ? @@ -2185,7 +2260,7 @@ function Init { SYNC_OPTS=$SYNC_OPTS"i" fi - if [ $stats -eq 1 ]; then + if [ $STATS -eq 1 ]; then SYNC_OPTS=$SYNC_OPTS" --stats" fi @@ -2194,11 +2269,6 @@ function Init { RsyncPatterns fi - ## Filenames for state files - if [ $_DRYRUN -eq 1 ]; then - dry_suffix="-dry" - fi - ## Conflict options if [ "$CONFLICT_BACKUP" != "no" ]; then INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[1]}${INITIATOR[4]}\"" @@ -2281,6 +2351,9 @@ function Usage { function SyncOnChanges { + local cmd= + local retval= + if ! type inotifywait > /dev/null 2>&1 ; then Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL" exit 1 @@ -2305,8 +2378,8 @@ function SyncOnChanges { Logger "#### Monitoring now." "NOTICE" inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & - OSYNC_SUB_PID=$! - wait $OSYNC_SUB_PID + #OSYNC_SUB_PID=$! Not used anymore with killchilds func + wait $! retval=$? if [ $retval == 0 ]; then Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE" @@ -2321,7 +2394,7 @@ function SyncOnChanges { } -stats=0 +STATS=0 PARTIAL=0 FORCE_UNLOCK=0 no_maxtime=0 @@ -2471,6 +2544,7 @@ opts="${opts# *}" GetLocalOS InitLocalOSSettings + CheckEnvironment PreInit Init PostInit