From ab4ed13d8b8b745844595c162fa863e7120e7613 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 25 May 2014 19:09:30 +0200 Subject: [PATCH] Improved remote deletion --- osync.sh | 78 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/osync.sh b/osync.sh index 7489332..426cfe9 100755 --- a/osync.sh +++ b/osync.sh @@ -4,7 +4,7 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=0.99preRC3 -PROGRAM_BUILD=2505201405 +PROGRAM_BUILD=2505201406 ## allow debugging from command line with preceding ocsync with DEBUG=yes if [ ! "$DEBUG" == "yes" ] @@ -334,9 +334,9 @@ function GetOperatingSystem fi fi fi - + REMOTE_OS_VAR=$(cat $RUN_DIR/osync_remote_os_$SCRIPT_PID) - + case $REMOTE_OS_VAR in *"Linux"*) REMOTE_OS="Linux" @@ -378,9 +378,8 @@ function WaitForTaskCompletion while eval $PROCESS_TEST > /dev/null do Spinner - sleep 1 EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) - if [ $(($EXEC_TIME % $KEEP_LOGGING)) -eq 0 ] + if [ $((($EXEC_TIME + 1) % $KEEP_LOGGING)) -eq 0 ] then Log "Current task still running." fi @@ -409,6 +408,7 @@ function WaitForTaskCompletion return 1 fi fi + sleep 1 done wait $child_pid return $? @@ -428,8 +428,7 @@ function WaitForCompletion while eval $PROCESS_TEST > /dev/null do Spinner - sleep 1 - if [ $(($SECONDS % $KEEP_LOGGING)) -eq 0 ] + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ] then Log "Current task still running." fi @@ -458,6 +457,7 @@ function WaitForCompletion return 1 fi fi + sleep 1 done wait $child_pid return $? @@ -487,7 +487,7 @@ function RunLocalCommand then Log "Command output:\n$(cat $RUN_DIR/osync_run_local_$SCRIPT_PID)" fi - + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ] then exit 1 @@ -705,11 +705,11 @@ function CheckMinimumSpace then LogError "There is not enough free space on master [$MASTER_SPACE KB]." fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost + CheckConnectivityRemoteHost eval "$SSH_CMD \"$COMMAND_SUDO df -P \\\"$SLAVE_SYNC_DIR\\\"\"" > $RUN_DIR/osync_slave_space_$SCRIPT_PID & child_pid=$! WaitForTaskCompletion $child_pid 0 1800 @@ -717,7 +717,7 @@ function CheckMinimumSpace else SLAVE_SPACE=$(df -P "$SLAVE_SYNC_DIR" | tail -1 | awk '{print $4}') fi - + if [ $SLAVE_SPACE -lt $MINIMUM_SPACE ] then LogError "There is not enough free space on slave [$SLAVE_SPACE KB]." @@ -822,7 +822,7 @@ function LockDirectories exit 1 fi fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -940,9 +940,9 @@ function tree_list if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f $RUN_DIR/osync_$2_$SCRIPT_PID ] then if [ $dryrun -eq 1 ] - then + then mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/dry-$2-$SYNC_ID" - else + else mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2-$SYNC_ID" fi echo "$3.success" > "$MASTER_LAST_ACTION" @@ -1020,7 +1020,7 @@ function sync_update ESC_DEST_DIR=$(EscapeSpaces "$MASTER_SYNC_DIR") BACKUP_DIR="$MASTER_BACKUP" fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -1062,12 +1062,11 @@ function sync_update # delete_local(replica dir, delete file list, delete dir) function _delete_local { - ## On every run, check wheter the next item is already deleted because it's included in a directory already deleted previous_file="" OLD_IFS=$IFS IFS=$'\r\n' - for files in $(cat "$2") + for files in $(cat "$MASTER_STATE_DIR/$2") do if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ] then @@ -1086,7 +1085,7 @@ function _delete_local then Log "Soft deleting $REPLICA_DIR$files" fi - + if [ $dryrun -ne 1 ] then mv "$REPLICA_DIR$files" "$REPLICA_DIR$3" @@ -1123,9 +1122,9 @@ function _delete_remote ## Anything beetween << ENDSSH and ENDSSH will be executed remotely # Additionnaly, we need to copy the deletetion list to the remote state folder - ESC_DEST_DIR="$(EscapeSpaces $SLAVE_STATE_DIR)" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"$2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID 2>&1" - eval $rsync_cmd + ESC_DEST_DIR="$(EscapeSpaces "$SLAVE_STATE_DIR")" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"$MASTER_STATE_DIR/$2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID 2>&1" + eval $rsync_cmd 2>> "$LOG_FILE" if [ $? != 0 ] then LogError "Cannot copy the deletion list to remote replica." @@ -1136,7 +1135,7 @@ function _delete_remote exit 1 fi -$SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DEBUG dryrun=$dryrun verbose=$verbose COMMAND_SUDO=$COMMAND_SUDO FILE_LIST=$(EscapeSpaces "$SLAVE_STATE_DIR/$(basename $2)") REPLICA_DIR="$(EscapeSpaces "$REPLICA_DIR") DELETE_DIR="$(EscapeSpaces "$DELETE_DIR") 'bash -s' << 'ENDSSH' > $RUN_DIR/osync_remote_deletion_$SCRIPT_PID 2>&1 & +$SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DEBUG dryrun=$dryrun verbose=$verbose COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$SLAVE_STATE_DIR/$2")" REPLICA_DIR="$(EscapeSpaces "$REPLICA_DIR")" DELETE_DIR="$(EscapeSpaces "$DELETE_DIR")" 'bash -s' << 'ENDSSH' > $RUN_DIR/osync_remote_deletion_$SCRIPT_PID 2>&1 & ## The following lines are executed remotely function Log @@ -1184,7 +1183,7 @@ $SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DE if [ $dryrun -ne 1 ] then - $COMMAND_SUDO mv "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR" + $COMMAND_SUDO mv "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR" if [ $? != 0 ] then LogError "Cannot move $REPLICA_DIR$files to deletion directory." @@ -1209,7 +1208,8 @@ $SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DE fi done ENDSSH - + ## Need to add a trivial sleep time to give ssh time to log to local file + sleep 5 exit $? } @@ -1219,16 +1219,16 @@ ENDSSH function deletion_propagation { Log "Propagating deletions to $1 replica." - + if [ "$1" == "master" ] then REPLICA_DIR="$MASTER_SYNC_DIR" DELETE_DIR="$MASTER_DELETE_DIR" if [ $dryrun -eq 1 ] then - DELETION_FILE_LIST="$MASTER_STATE_DIR/dry-slave-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="dry-slave-deleted-list-$SYNC_ID" else - DELETION_FILE_LIST="$MASTER_STATE_DIR/slave-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="slave-deleted-list-$SYNC_ID" fi _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & @@ -1242,15 +1242,15 @@ function deletion_propagation LogError "Deletion on replica $1 failed." echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" exit 1 - fi + fi else REPLICA_DIR="$SLAVE_SYNC_DIR" DELETE_DIR="$SLAVE_DELETE_DIR" if [ $dryrun -eq 1 ] then - DELETION_FILE_LIST="$MASTER_STATE_DIR/dry-master-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="dry-master-deleted-list-$SYNC_ID" else - DELETION_FILE_LIST="$MASTER_STATE_DIR/master-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="master-deleted-list-$SYNC_ID" fi if [ "$REMOTE_SYNC" == "yes" ] @@ -1264,12 +1264,16 @@ function deletion_propagation retval=$? if [ $retval == 0 ] then + if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] && [ $verbose -eq 1 ] + then + Log "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" + fi echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" else LogError "Deletion on remote system failed." if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] then - LogError "$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" + LogError "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" fi echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" exit 1 @@ -1401,7 +1405,7 @@ function Sync if [ $dryrun -ne 1 ] then echo "0" > "$MASTER_RESUME_COUNT" - fi + fi } function SoftDelete @@ -1436,7 +1440,7 @@ function SoftDelete then LogError "Warning: Master replica conflict backup dir [$MASTER_SYNC_DIR$MASTER_BACKUP_DIR] isn't writable. Cannot clean old files." fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -1602,7 +1606,7 @@ function Init else FIND_CMD=find fi - + if [ "$REMOTE_OS" == "msys" ] then REMOTE_FIND_CMD=$(dirname $BASH)/find @@ -1663,7 +1667,7 @@ function Init MASTER_DELETE_DIR="$OSYNC_DIR/deleted" SLAVE_BACKUP_DIR="$OSYNC_DIR/backups" SLAVE_DELETE_DIR="$OSYNC_DIR/deleted" - + ## SSH compression if [ "$SSH_COMPRESSION" != "no" ] then @@ -1743,7 +1747,7 @@ function Init ## Set sync only function arguments for rsync SYNC_OPTS="-u" - + if [ $verbose -eq 1 ] then SYNC_OPTS=$SYNC_OPTS"i" @@ -1818,7 +1822,7 @@ function SyncOnChanges exit 1 fi - Log "#### Running Osync in file monitor mode." + Log "#### Running Osync in file monitor mode." while true do