From 8214f151db12c2c5275d59b5167f14f4e4d7caf1 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 25 May 2014 17:51:05 +0200 Subject: [PATCH 1/5] Revert typos --- CHANGELOG.md | 2 +- osync-srv | 7 +- osync.sh | 311 +++++++++++++++++++++++++++++++++++++++++---------- sync.conf | 13 ++- 4 files changed, 263 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24042d4..773752d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,11 +19,11 @@ KNOWN ISSUES UNDER WORK ---------- -!- Better deletion propagation (again). Using rsync for deletion propagation is definitly not working in all cases (especially empty sub directories) RECENT CHANGES -------------- +- Fixed deletion propagation (again). Rsync is definitly not designed to delete a list of files / folders. Rsync replaced by rm function which downloads deletion list to remote system. - Added path detection for exclude list file - Added a simple init script working for RHEL / CentOS and an install script - Fixed an issue with MacOSX using rsync -E differently than other *nix (Thanks to Pierre Clement) diff --git a/osync-srv b/osync-srv index 4309eaf..1afe07c 100755 --- a/osync-srv +++ b/osync-srv @@ -92,9 +92,11 @@ status() { if [ ! -f $pidfile-* ] then echo "Cannot find any running osync instance." - exit + exit 1 fi + errno=0 + for pfile in $pidfile-* do if ps -p$(cat $pfile) > /dev/null 2>&1 @@ -102,8 +104,11 @@ status() { echo "$prog instance $(basename $pfile) is running (pid $(cat $pfile))" else echo "$pfile is dead but lockfile exists." + $errno=1 fi done + + exit $errno } case "$1" in diff --git a/osync.sh b/osync.sh index 1f574f9..7489332 100755 --- a/osync.sh +++ b/osync.sh @@ -4,7 +4,7 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=0.99preRC3 -PROGRAM_BUILD=2205201401 +PROGRAM_BUILD=2505201405 ## allow debugging from command line with preceding ocsync with DEBUG=yes if [ ! "$DEBUG" == "yes" ] @@ -983,7 +983,12 @@ function delete_list eval $cmd echo "$1-replica-deleted-list.success" > "$MASTER_LAST_ACTION" else - touch "$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID" + if [ $dryrun -eq 1 ] + then + touch "$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID" + else + touch "$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID" + fi echo "$1-replica-deleted-list.empty" > "$MASTER_LAST_ACTION" fi } @@ -991,6 +996,15 @@ function delete_list # sync_update(source replica, destination replica) function sync_update { + if [ $dryrun -eq 1 ] + then + delfname1="dry-$1-deleted-list-$SYNC_ID" + delfname2="dry-$2-deleted-list-$SYNC_ID" + else + delfname1="$1-deleted-list-$SYNC_ID" + delfname2="$2-deleted-list-$SYNC_ID" + fi + Log "Updating $2 replica." if [ "$1" == "master" ] then @@ -1013,12 +1027,12 @@ function sync_update CheckConnectivityRemoteHost if [ "$1" == "master" ] then - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -ui --stats -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\" --exclude-from=\"$MASTER_STATE_DIR/$2-deleted-list-$SYNC_ID\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -ui --stats -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\" --exclude-from=\"$MASTER_STATE_DIR/$2-deleted-list-$SYNC_ID\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" fi else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -ui --stats $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\" --exclude-from=\"$MASTER_STATE_DIR/$2-deleted-list-$SYNC_ID\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" fi LogDebug "RSYNC_CMD: $rsync_cmd" eval "$rsync_cmd" @@ -1045,58 +1059,221 @@ function sync_update fi } -# delete_propagation(source replica, destination replica) +# delete_local(replica dir, delete file list, delete dir) +function _delete_local +{ + + ## On every run, check wheter the next item is already deleted because it's included in a directory already deleted + previous_file="" + OLD_IFS=$IFS + IFS=$'\r\n' + for files in $(cat "$2") + do + if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ] + then + if [ "$SOFT_DELETE" != "no" ] + then + if [ ! -d "$REPLICA_DIR$3" ] + then + mkdir -p "$REPLICA_DIR$3" + if [ $? != 0 ] + then + LogError "Cannot create replica deletion directory." + fi + fi + + if [ $verbose -eq 1 ] + then + Log "Soft deleting $REPLICA_DIR$files" + fi + + if [ $dryrun -ne 1 ] + then + mv "$REPLICA_DIR$files" "$REPLICA_DIR$3" + if [ $? != 0 ] + then + LogError "Cannot move $REPLICA_DIR$files to deletion directory." + fi + fi + else + if [ $verbose -eq 1 ] + then + Log "Deleting $REPLICA_DIR$files" + fi + + if [ $dryrun -ne 1 ] + then + rm -rf "$REPLICA_DIR$files" + if [ $? != 0 ] + then + LogError "Cannot delete $REPLICA_DIR$files" + fi + fi + fi + previous_file="$files" + fi + done + IFS=$OLD_IFS +} + +# delete_remote(replica dir, delete file list, delete dir) +function _delete_remote +{ + ## This is a special coded function. Need to redelcare local functions on remote host, passing all needed variables as escaped arguments to ssh command. + ## Anything beetween << ENDSSH and ENDSSH will be executed remotely + + # Additionnaly, we need to copy the deletetion list to the remote state folder + ESC_DEST_DIR="$(EscapeSpaces $SLAVE_STATE_DIR)" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"$2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID 2>&1" + eval $rsync_cmd + if [ $? != 0 ] + then + LogError "Cannot copy the deletion list to remote replica." + if [ -f $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID ] + then + LogError "$(cat $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID)" + fi + exit 1 + fi + +$SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DEBUG dryrun=$dryrun verbose=$verbose COMMAND_SUDO=$COMMAND_SUDO FILE_LIST=$(EscapeSpaces "$SLAVE_STATE_DIR/$(basename $2)") REPLICA_DIR="$(EscapeSpaces "$REPLICA_DIR") DELETE_DIR="$(EscapeSpaces "$DELETE_DIR") 'bash -s' << 'ENDSSH' > $RUN_DIR/osync_remote_deletion_$SCRIPT_PID 2>&1 & + + ## The following lines are executed remotely + function Log + { + if [ $sync_on_changes -eq 1 ] + then + prefix="$(date) - " + else + prefix="R-TIME: $SECONDS - " + fi + + if [ $silent -eq 0 ] + then + echo -e "$prefix$1" + fi + } + + function LogError + { + Log "$1" + error_alert=1 + } + + ## On every run, check wheter the next item is already deleted because it's included in a directory already deleted + previous_file="" + for files in $(cat "$FILE_LIST") + do + if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ] + then + if [ ! -d "$REPLICA_DIR$DELETE_DIR" ] + then + $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR" + if [ $? != 0 ] + then + LogError "Cannot create replica deletion directory." + fi + fi + + if [ "$SOFT_DELETE" != "no" ] + then + if [ $verbose -eq 1 ] + then + Log "Soft deleting $REPLICA_DIR$files" + fi + + if [ $dryrun -ne 1 ] + then + $COMMAND_SUDO mv "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR" + if [ $? != 0 ] + then + LogError "Cannot move $REPLICA_DIR$files to deletion directory." + fi + fi + else + if [ $verbose -eq 1 ] + then + Log "Deleting $REPLICA_DIR$files" + fi + + if [ $dryrun -ne 1 ] + then + $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + if [ $? != 0 ] + then + LogError "Cannot delete $REPLICA_DIR$files" + fi + fi + fi + previous_file="$files" + fi + done +ENDSSH + + exit $? +} + + +# delete_propagation(replica name) +# replica name = "master" / "slave" function deletion_propagation { - Log "Propagating deletions to $2 replica." + Log "Propagating deletions to $1 replica." + if [ "$1" == "master" ] then - SOURCE_DIR="$MASTER_SYNC_DIR" - ESC_SOURCE_DIR=$(EscapeSpaces "$MASTER_SYNC_DIR") - DEST_DIR="$SLAVE_SYNC_DIR" - ESC_DEST_DIR=$(EscapeSpaces "$SLAVE_SYNC_DIR") - DELETE_DIR="$SLAVE_DELETE" - else - SOURCE_DIR="$SLAVE_SYNC_DIR" - ESC_SOURCE_DIR=$(EscapeSpaces "$SLAVE_SYNC_DIR") - DEST_DIR="$MASTER_SYNC_DIR" - ESC_DEST_DIR=$(EscapeSpaces "$MASTER_SYNC_DIR") - DELETE_DIR="$MASTER_DELETE" - fi - if [ "$REMOTE_SYNC" == "yes" ] - then - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - if [ "$1" == "master" ] + REPLICA_DIR="$MASTER_SYNC_DIR" + DELETE_DIR="$MASTER_DELETE_DIR" + if [ $dryrun -eq 1 ] then - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -ui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" + DELETION_FILE_LIST="$MASTER_STATE_DIR/dry-slave-deleted-list-$SYNC_ID" else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -ui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" + DELETION_FILE_LIST="$MASTER_STATE_DIR/slave-deleted-list-$SYNC_ID" fi - else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -ui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" - fi - LogDebug "RSYNC_CMD: $rsync_cmd" - eval "$rsync_cmd" - child_pid=$! - WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 - retval=$? - if [ $verbose -eq 1 ] && [ -f $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID ] - then - Log "List:\n$(cat $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID)" - fi - if [ $retval != 0 ] && [ $retval != 24 ] - then - if [ $verbose -eq 0 ] && [ -f $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID ] - then - LogError "Rsync output:\n$(cat $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID)" - fi - LogError "Deletion on $2 failed." - echo "delete-propagation-$2.fail" > "$MASTER_LAST_ACTION" - exit 1 + _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME + retval=$? + if [ $retval == 0 ] + then + echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" + else + LogError "Deletion on replica $1 failed." + echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" + exit 1 + fi else - echo "delete-propagation-$2.success" > "$MASTER_LAST_ACTION" + REPLICA_DIR="$SLAVE_SYNC_DIR" + DELETE_DIR="$SLAVE_DELETE_DIR" + if [ $dryrun -eq 1 ] + then + DELETION_FILE_LIST="$MASTER_STATE_DIR/dry-master-deleted-list-$SYNC_ID" + else + DELETION_FILE_LIST="$MASTER_STATE_DIR/master-deleted-list-$SYNC_ID" + fi + + if [ "$REMOTE_SYNC" == "yes" ] + then + _delete_remote "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + else + _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME + retval=$? + if [ $retval == 0 ] + then + echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" + else + LogError "Deletion on remote system failed." + if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] + then + LogError "$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" + fi + echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" + exit 1 + fi fi } @@ -1197,12 +1374,12 @@ function Sync fi if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-slave-replica.success" ] || [ "$resume_sync" == "update-master-replica.success" ] || [ "$resume_sync" == "delete-propagation-slave.fail" ] then - deletion_propagation master slave + deletion_propagation slave resume_sync="resumed" fi if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "delete-propagation-slave.success" ] || [ "$resume_sync" == "delete-propagation-master.fail" ] then - deletion_propagation slave master + deletion_propagation master resume_sync="resumed" fi if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "delete-propagation-master.success" ] || [ "$resume_sync" == "master-replica-tree-after.fail" ] @@ -1408,9 +1585,9 @@ function Init then if [ -w /var/log ] then - LOG_FILE=/var/log/osync_$PROGRAM_VERSION-$SYNC_ID.log + LOG_FILE=/var/log/osync_$SYNC_ID.log else - LOG_FILE=./osync_$PROGRAM_VERSION-$SYNC_ID.log + LOG_FILE=./osync_$SYNC_ID.log fi else LOG_FILE="$LOGFILE" @@ -1499,6 +1676,7 @@ function Init if [ "$REMOTE_SYNC" == "yes" ] then SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" fi @@ -1563,6 +1741,19 @@ function Init RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" fi + ## Set sync only function arguments for rsync + SYNC_OPTS="-u" + + if [ $verbose -eq 1 ] + then + SYNC_OPTS=$SYNC_OPTS"i" + fi + + if [ $stats -eq 1 ] + then + SYNC_OPTS=$SYNC_OPTS" --stats" + fi + ## Conflict options if [ "$CONFLICT_BACKUP" != "no" ] then @@ -1578,16 +1769,6 @@ function Init SLAVE_BACKUP= fi - ## Soft delete options - if [ "$SOFT_DELETE" != "no" ] - then - MASTER_DELETE="--backup --backup-dir=\"$MASTER_DELETE_DIR\"" - SLAVE_DELETE="--backup --backup-dir=\"$SLAVE_DELETE_DIR\"" - else - MASTER_DELETE= - SLAVE_DELETE= - fi - ## Add Rsync exclude patterns RsyncExcludePattern ## Add Rsync exclude from file @@ -1616,6 +1797,7 @@ function Usage echo "--dry Will run osync without actually doing anything; just testing" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--verbose Increases output" + echo "--stats Adds transfer statistics" echo "--no-maxtime Disables any soft and hard execution time checks" echo "--force-unlock Will override any existing active or dead locks on master and slave replica" echo "--on-changes Will launch a sync as soon as there is some file activity on master replica" @@ -1667,6 +1849,7 @@ function SyncOnChanges dryrun=0 silent=0 verbose=0 +stats=0 force_unlock=0 no_maxtime=0 # Alert flags @@ -1699,6 +1882,10 @@ do verbose=1 opts=$opts" --verbose" ;; + --stats) + stats=1 + opts=$opts" --stats" + ;; --force-unlock) force_unlock=1 opts=$opts" --force-unlock" diff --git a/sync.conf b/sync.conf index 0bc835f..79d0247 100755 --- a/sync.conf +++ b/sync.conf @@ -2,7 +2,7 @@ ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2205201401 +###### Config file rev 2505201402 ## ---------- GENERAL OPTIONS @@ -12,7 +12,7 @@ SYNC_ID="sync_test" ## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one. MASTER_SYNC_DIR="/home/git/osync/dir1" SLAVE_SYNC_DIR="/home/git/osync/dir2" -#SLAVE_SYNC_DIR="ssh://user@host.com:22//path/to/dir2" +#SLAVE_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir2" ## If slave replica is a remote directory, you must specifiy a RSA key (please use full path). Please see documentation for further information. SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa" @@ -24,7 +24,8 @@ LOGFILE="" ## List of directories to exclude from sync on both sides (rsync patterns, wildcards work). ## Paths are relative to sync dirs. List elements are separated by a semicolon. -RSYNC_EXCLUDE_PATTERN="tmp;archives" +RSYNC_EXCLUDE_PATTERN="" +#RSYNC_EXCLUDE_PATTERN="tmp;archives" ## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. ## This file has to be in the same directory as the config file ## Paths are relative to sync dirs. One element per line. @@ -39,7 +40,7 @@ MINIMUM_SPACE=10240 ## Bandwidth limit Kbytes / second. Leave 0 to disable limitation BANDWIDTH=0 -## If enabled, synchronization will be processed as superuser. See documentation for /etc/sudoers file configuration. +## If enabled, synchronization on remote system will be processed as superuser. See documentation for /etc/sudoers file configuration. SUDO_EXEC=no ## Paranoia option. Don't change this unless you read the documentation. RSYNC_EXECUTABLE=rsync @@ -110,8 +111,8 @@ FORCE_STRANGER_LOCK_RESUME=no DESTINATION_MAILS="your@alert.tld" ## Windows (MSYS environment) only mail options (used with sendemail.exe from Brandon Zehm) -SENDER_MAIL="alert@your.system" -SMTP_SERVER=smtp.your.isp.com +SENDER_MAIL="alert@your.system.tld" +SMTP_SERVER=smtp.your.isp.tld SMTP_USER= SMTP_PASSWORD= From ab4ed13d8b8b745844595c162fa863e7120e7613 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 25 May 2014 19:09:30 +0200 Subject: [PATCH 2/5] Improved remote deletion --- osync.sh | 78 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/osync.sh b/osync.sh index 7489332..426cfe9 100755 --- a/osync.sh +++ b/osync.sh @@ -4,7 +4,7 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=0.99preRC3 -PROGRAM_BUILD=2505201405 +PROGRAM_BUILD=2505201406 ## allow debugging from command line with preceding ocsync with DEBUG=yes if [ ! "$DEBUG" == "yes" ] @@ -334,9 +334,9 @@ function GetOperatingSystem fi fi fi - + REMOTE_OS_VAR=$(cat $RUN_DIR/osync_remote_os_$SCRIPT_PID) - + case $REMOTE_OS_VAR in *"Linux"*) REMOTE_OS="Linux" @@ -378,9 +378,8 @@ function WaitForTaskCompletion while eval $PROCESS_TEST > /dev/null do Spinner - sleep 1 EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) - if [ $(($EXEC_TIME % $KEEP_LOGGING)) -eq 0 ] + if [ $((($EXEC_TIME + 1) % $KEEP_LOGGING)) -eq 0 ] then Log "Current task still running." fi @@ -409,6 +408,7 @@ function WaitForTaskCompletion return 1 fi fi + sleep 1 done wait $child_pid return $? @@ -428,8 +428,7 @@ function WaitForCompletion while eval $PROCESS_TEST > /dev/null do Spinner - sleep 1 - if [ $(($SECONDS % $KEEP_LOGGING)) -eq 0 ] + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ] then Log "Current task still running." fi @@ -458,6 +457,7 @@ function WaitForCompletion return 1 fi fi + sleep 1 done wait $child_pid return $? @@ -487,7 +487,7 @@ function RunLocalCommand then Log "Command output:\n$(cat $RUN_DIR/osync_run_local_$SCRIPT_PID)" fi - + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ] then exit 1 @@ -705,11 +705,11 @@ function CheckMinimumSpace then LogError "There is not enough free space on master [$MASTER_SPACE KB]." fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost + CheckConnectivityRemoteHost eval "$SSH_CMD \"$COMMAND_SUDO df -P \\\"$SLAVE_SYNC_DIR\\\"\"" > $RUN_DIR/osync_slave_space_$SCRIPT_PID & child_pid=$! WaitForTaskCompletion $child_pid 0 1800 @@ -717,7 +717,7 @@ function CheckMinimumSpace else SLAVE_SPACE=$(df -P "$SLAVE_SYNC_DIR" | tail -1 | awk '{print $4}') fi - + if [ $SLAVE_SPACE -lt $MINIMUM_SPACE ] then LogError "There is not enough free space on slave [$SLAVE_SPACE KB]." @@ -822,7 +822,7 @@ function LockDirectories exit 1 fi fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -940,9 +940,9 @@ function tree_list if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f $RUN_DIR/osync_$2_$SCRIPT_PID ] then if [ $dryrun -eq 1 ] - then + then mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/dry-$2-$SYNC_ID" - else + else mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2-$SYNC_ID" fi echo "$3.success" > "$MASTER_LAST_ACTION" @@ -1020,7 +1020,7 @@ function sync_update ESC_DEST_DIR=$(EscapeSpaces "$MASTER_SYNC_DIR") BACKUP_DIR="$MASTER_BACKUP" fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -1062,12 +1062,11 @@ function sync_update # delete_local(replica dir, delete file list, delete dir) function _delete_local { - ## On every run, check wheter the next item is already deleted because it's included in a directory already deleted previous_file="" OLD_IFS=$IFS IFS=$'\r\n' - for files in $(cat "$2") + for files in $(cat "$MASTER_STATE_DIR/$2") do if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ] then @@ -1086,7 +1085,7 @@ function _delete_local then Log "Soft deleting $REPLICA_DIR$files" fi - + if [ $dryrun -ne 1 ] then mv "$REPLICA_DIR$files" "$REPLICA_DIR$3" @@ -1123,9 +1122,9 @@ function _delete_remote ## Anything beetween << ENDSSH and ENDSSH will be executed remotely # Additionnaly, we need to copy the deletetion list to the remote state folder - ESC_DEST_DIR="$(EscapeSpaces $SLAVE_STATE_DIR)" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"$2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID 2>&1" - eval $rsync_cmd + ESC_DEST_DIR="$(EscapeSpaces "$SLAVE_STATE_DIR")" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"$MASTER_STATE_DIR/$2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID 2>&1" + eval $rsync_cmd 2>> "$LOG_FILE" if [ $? != 0 ] then LogError "Cannot copy the deletion list to remote replica." @@ -1136,7 +1135,7 @@ function _delete_remote exit 1 fi -$SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DEBUG dryrun=$dryrun verbose=$verbose COMMAND_SUDO=$COMMAND_SUDO FILE_LIST=$(EscapeSpaces "$SLAVE_STATE_DIR/$(basename $2)") REPLICA_DIR="$(EscapeSpaces "$REPLICA_DIR") DELETE_DIR="$(EscapeSpaces "$DELETE_DIR") 'bash -s' << 'ENDSSH' > $RUN_DIR/osync_remote_deletion_$SCRIPT_PID 2>&1 & +$SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DEBUG dryrun=$dryrun verbose=$verbose COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$SLAVE_STATE_DIR/$2")" REPLICA_DIR="$(EscapeSpaces "$REPLICA_DIR")" DELETE_DIR="$(EscapeSpaces "$DELETE_DIR")" 'bash -s' << 'ENDSSH' > $RUN_DIR/osync_remote_deletion_$SCRIPT_PID 2>&1 & ## The following lines are executed remotely function Log @@ -1184,7 +1183,7 @@ $SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DE if [ $dryrun -ne 1 ] then - $COMMAND_SUDO mv "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR" + $COMMAND_SUDO mv "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR" if [ $? != 0 ] then LogError "Cannot move $REPLICA_DIR$files to deletion directory." @@ -1209,7 +1208,8 @@ $SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DE fi done ENDSSH - + ## Need to add a trivial sleep time to give ssh time to log to local file + sleep 5 exit $? } @@ -1219,16 +1219,16 @@ ENDSSH function deletion_propagation { Log "Propagating deletions to $1 replica." - + if [ "$1" == "master" ] then REPLICA_DIR="$MASTER_SYNC_DIR" DELETE_DIR="$MASTER_DELETE_DIR" if [ $dryrun -eq 1 ] then - DELETION_FILE_LIST="$MASTER_STATE_DIR/dry-slave-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="dry-slave-deleted-list-$SYNC_ID" else - DELETION_FILE_LIST="$MASTER_STATE_DIR/slave-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="slave-deleted-list-$SYNC_ID" fi _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & @@ -1242,15 +1242,15 @@ function deletion_propagation LogError "Deletion on replica $1 failed." echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" exit 1 - fi + fi else REPLICA_DIR="$SLAVE_SYNC_DIR" DELETE_DIR="$SLAVE_DELETE_DIR" if [ $dryrun -eq 1 ] then - DELETION_FILE_LIST="$MASTER_STATE_DIR/dry-master-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="dry-master-deleted-list-$SYNC_ID" else - DELETION_FILE_LIST="$MASTER_STATE_DIR/master-deleted-list-$SYNC_ID" + DELETION_FILE_LIST="master-deleted-list-$SYNC_ID" fi if [ "$REMOTE_SYNC" == "yes" ] @@ -1264,12 +1264,16 @@ function deletion_propagation retval=$? if [ $retval == 0 ] then + if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] && [ $verbose -eq 1 ] + then + Log "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" + fi echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" else LogError "Deletion on remote system failed." if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] then - LogError "$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" + LogError "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" fi echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" exit 1 @@ -1401,7 +1405,7 @@ function Sync if [ $dryrun -ne 1 ] then echo "0" > "$MASTER_RESUME_COUNT" - fi + fi } function SoftDelete @@ -1436,7 +1440,7 @@ function SoftDelete then LogError "Warning: Master replica conflict backup dir [$MASTER_SYNC_DIR$MASTER_BACKUP_DIR] isn't writable. Cannot clean old files." fi - + if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -1602,7 +1606,7 @@ function Init else FIND_CMD=find fi - + if [ "$REMOTE_OS" == "msys" ] then REMOTE_FIND_CMD=$(dirname $BASH)/find @@ -1663,7 +1667,7 @@ function Init MASTER_DELETE_DIR="$OSYNC_DIR/deleted" SLAVE_BACKUP_DIR="$OSYNC_DIR/backups" SLAVE_DELETE_DIR="$OSYNC_DIR/deleted" - + ## SSH compression if [ "$SSH_COMPRESSION" != "no" ] then @@ -1743,7 +1747,7 @@ function Init ## Set sync only function arguments for rsync SYNC_OPTS="-u" - + if [ $verbose -eq 1 ] then SYNC_OPTS=$SYNC_OPTS"i" @@ -1818,7 +1822,7 @@ function SyncOnChanges exit 1 fi - Log "#### Running Osync in file monitor mode." + Log "#### Running Osync in file monitor mode." while true do From d8ff286b32d10c1c9c5eb46e8bb39ee71c87f57c Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 27 May 2014 00:50:46 +0200 Subject: [PATCH 3/5] Code cleanup --- CHANGELOG.md | 8 +- install.sh | 4 +- osync-srv | 3 +- osync.sh | 403 ++++++++++++++++++++++++++++++--------------------- sync.conf | 4 +- 5 files changed, 247 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 773752d..e05180a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ KNOWN ISSUES ------------ - Still need more testing on BSD, MacOSX and Windows MSYS +- Cannot finish sync if one replica contains a directory and the other replica contains a file named the same way (Unix doesn't allow this) UNDER WORK ---------- @@ -23,9 +24,13 @@ UNDER WORK RECENT CHANGES -------------- +- Added some additionnal checks for *BSD and MacOS environments +- Changed /bin/bash to /usr/bin/env bash for sanity on other systems, also check for bash presence before running +- Changed default behavior for quick sync tasks: Will try to resume failed sync tasks once +- Some code cleanup for state filenames and sync action names - Fixed deletion propagation (again). Rsync is definitly not designed to delete a list of files / folders. Rsync replaced by rm function which downloads deletion list to remote system. - Added path detection for exclude list file -- Added a simple init script working for RHEL / CentOS and an install script +- Added a simple init script and an install script - Fixed an issue with MacOSX using rsync -E differently than other *nix (Thanks to Pierre Clement) - Multislave asynchronous task support (Thanks to Ulrich Norbisrath) - This breaks compat with elder osync runs. Add the SYNC_ID suffix to elder state files to keep deleted file information. @@ -39,7 +44,6 @@ RECENT CHANGES - Fixed a nasty bug preventing writing lock files on remote system as superuser - Gzipped logs are now deleted once sent - Fixed some typos (thanks to Pavel Kiryukhin) -- Added a simple RHEL / CentOS compatible init script - Fixed a bug with double trailing slashes in certain sceanrios - Sync execution don't fails anymore if files vanish during execution, also vanished files get logged - Add eventual "comm -23" replacement by "grep -F -x -v -f" to enhance compatibility with other platforms (comm is still much faster than grep, so we keep it) diff --git a/install.sh b/install.sh index 8b38309..4b3a378 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash + +SCRIPT_BUILD=2605201401 ## Osync daemon install script ## Tested on RHEL / CentOS 6 diff --git a/osync-srv b/osync-srv index 1afe07c..df59df4 100755 --- a/osync-srv +++ b/osync-srv @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # osync two way directory sync tool # @@ -12,6 +12,7 @@ progpath=/usr/local/bin confdir=/etc/osync pidfile=/var/run/$prog lockfile=/var/lock/subsys/$prog +SCRIPT_BUILD=2605201401 if [ ! -f $progpath/$progexec ] && [ ! -f $progexec ] then diff --git a/osync.sh b/osync.sh index 426cfe9..52f79ee 100755 --- a/osync.sh +++ b/osync.sh @@ -1,10 +1,16 @@ -#!/bin/bash +#!/usr/bin/env bash PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=0.99preRC3 -PROGRAM_BUILD=2505201406 +PROGRAM_VERSION=0.99RC3 +PROGRAM_BUILD=2605201403 + +if ! type -p "$BASH" > /dev/null +then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 +fi ## allow debugging from command line with preceding ocsync with DEBUG=yes if [ ! "$DEBUG" == "yes" ] @@ -276,7 +282,7 @@ function CheckEnvironment fi } -function GetOperatingSystem +function GetLocalOS { LOCAL_OS_VAR=$(uname -spio 2>&1) if [ $? != 0 ] @@ -292,8 +298,8 @@ function GetOperatingSystem *"Linux"*) LOCAL_OS="Linux" ;; - *"FreeBSD"*) - LOCAL_OS="FreeBSD" + *"BSD"*) + LOCAL_OS="BSD" ;; *"MINGW32"*) LOCAL_OS="msys" @@ -307,7 +313,10 @@ function GetOperatingSystem ;; esac LogDebug "Local OS: [$LOCAL_OS_VAR]." +} +function GetRemoteOS +{ if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -341,8 +350,8 @@ function GetOperatingSystem *"Linux"*) REMOTE_OS="Linux" ;; - *"FreeBSD"*) - REMOTE_OS="FreeBSD" + *"BSD"*) + REMOTE_OS="BSD" ;; *"MINGW32"*) REMOTE_OS="msys" @@ -371,11 +380,11 @@ function WaitForTaskCompletion SECONDS_BEGIN=$SECONDS if [ "$LOCAL_OS" == "msys" ] then - PROCESS_TEST="ps -a | awk '{\$1=\$1}\$1' | awk '{print \$1}' | grep $1" + PROCESS_TEST_CMD="ps -a | awk '{\$1=\$1}\$1' | awk '{print \$1}' | grep $1" else - PROCESS_TEST="ps -p$1" + PROCESS_TEST_CMD="ps -p$1" fi - while eval $PROCESS_TEST > /dev/null + while eval $PROCESS_TEST_CMD > /dev/null do Spinner EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) @@ -557,12 +566,7 @@ function CheckConnectivityRemoteHost { if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ] then - if [ "$LOCAL_OS" == "msys" ] - then - ping -n 2 $REMOTE_HOST > /dev/null 2>&1 - else - ping -c 2 $REMOTE_HOST > /dev/null 2>&1 - fi + $PING_CMD $REMOTE_HOST > /dev/null 2>&1 if [ $? != 0 ] then LogError "Cannot ping $REMOTE_HOST" @@ -580,12 +584,7 @@ function CheckConnectivity3rdPartyHosts IFS=$' \t\n' for i in $REMOTE_3RD_PARTY_HOSTS do - if [ "$LOCAL_OS" == "msys" ] - then - ping -n 2 $i > /dev/null 2>&1 - else - ping -c 2 $i > /dev/null 2>&1 - fi + $PING_CMD $i > /dev/null 2>&1 if [ $? != 0 ] then Log "Cannot ping 3rd party host $i" @@ -917,11 +916,11 @@ function UnlockDirectories ## The same applies for slave sync dir..............................................T.H.I.S..I.S..A..P.R.O.G.R.A.M.M.I.N.G..N.I.G.H.T.M.A.R.E -## tree_list(replica_path, tree_file, current_action) Creates a list of files in replica_path and stores it's action in $STATE_DIR/last-action +## tree_list(replica_path, replica type, tree_filename) Creates a list of files in replica_path for replica type (master/slave) in filename $3 function tree_list { Log "Creating $2 replica file list [$1]." - if [ "$REMOTE_SYNC" == "yes" ] && [[ "$2" == "slave"* ]] + if [ "$REMOTE_SYNC" == "yes" ] && [ "$2" == "slave" ] then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -939,72 +938,41 @@ function tree_list ## Retval 24 = some files vanished while creating list if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f $RUN_DIR/osync_$2_$SCRIPT_PID ] then - if [ $dryrun -eq 1 ] - then - mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/dry-$2-$SYNC_ID" - else - mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2-$SYNC_ID" - fi - echo "$3.success" > "$MASTER_LAST_ACTION" - + mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2$3" + return $? else LogError "Cannot create replica file list." - echo "$3.fail" > "$MASTER_LAST_ACTION" - exit 1 + exit $retval fi } -# delete_list(replica): Creates a list of files vanished from last run on replica $1 +# delete_list(replica, tree-file-after, tree-file-current, deleted-list-file): Creates a list of files vanished from last run on replica $1 (master/slave) function delete_list { Log "Creating $1 replica deleted file list." if [ -f "$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID" ] then - if [ $dryrun -eq 1 ] + ## Same functionnality, comm is much faster than grep but is not available on every platform + if type -p comm > /dev/null 2>&1 then - ## Same functionnality, comm is much faster than grep but is not available on every platform - if type -p comm > /dev/null 2>&1 - then - cmd="comm -23 \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" \"$MASTER_STATE_DIR/dry-$1-tree-current-$SYNC_ID\" > \"$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID\"" - else - ## The || : forces the command to have a good result - cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/dry-$1-tree-current-$SYNC_ID\" \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" || : > \"$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID\"" - fi + cmd="comm -23 \"$MASTER_STATE_DIR/$1$2\" \"$MASTER_STATE_DIR/$1$3\" > \"$MASTER_STATE_DIR/$1$4\"" else - if type -p comm > /dev/null 2>&1 - then - cmd="comm -23 \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" \"$MASTER_STATE_DIR/$1-tree-current-$SYNC_ID\" > \"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\"" - else - cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/$1-tree-current-$SYNC_ID\" \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" || : > \"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\"" - fi + ## The || : forces the command to have a good result + cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/$1$3\" \"$MASTER_STATE_DIR/$1$2\" || : > \"$MASTER_STATE_DIR/$1$4\"" fi LogDebug "CMD: $cmd" eval $cmd - echo "$1-replica-deleted-list.success" > "$MASTER_LAST_ACTION" + return $? else - if [ $dryrun -eq 1 ] - then - touch "$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID" - else - touch "$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID" - fi - echo "$1-replica-deleted-list.empty" > "$MASTER_LAST_ACTION" + touch "$MASTER_STATE_DIR/$1$4" + return $? fi } -# sync_update(source replica, destination replica) +# sync_update(source replica, destination replica, delete_list_filename) function sync_update { - if [ $dryrun -eq 1 ] - then - delfname1="dry-$1-deleted-list-$SYNC_ID" - delfname2="dry-$2-deleted-list-$SYNC_ID" - else - delfname1="$1-deleted-list-$SYNC_ID" - delfname2="$2-deleted-list-$SYNC_ID" - fi - Log "Updating $2 replica." if [ "$1" == "master" ] then @@ -1027,12 +995,12 @@ function sync_update CheckConnectivityRemoteHost if [ "$1" == "master" ] then - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1$3\" --exclude-from=\"$MASTER_STATE_DIR/$2$3\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$2$3\" --exclude-from=\"$MASTER_STATE_DIR/$1$3\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" fi else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1$3\" --exclude-from=\"$MASTER_STATE_DIR/$2$3\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" fi LogDebug "RSYNC_CMD: $rsync_cmd" eval "$rsync_cmd" @@ -1051,11 +1019,10 @@ function sync_update then LogError "Rsync output:\n$(cat $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID)" fi - echo "update-$2-replica.fail" > "$MASTER_LAST_ACTION" - exit 1 + exit $retval else Log "Updating $2 replica succeded." - echo "update-$2-replica.success" > "$MASTER_LAST_ACTION" + return 0 fi } @@ -1214,7 +1181,7 @@ ENDSSH } -# delete_propagation(replica name) +# delete_propagation(replica name, deleted_list_filename) # replica name = "master" / "slave" function deletion_propagation { @@ -1224,40 +1191,25 @@ function deletion_propagation then REPLICA_DIR="$MASTER_SYNC_DIR" DELETE_DIR="$MASTER_DELETE_DIR" - if [ $dryrun -eq 1 ] - then - DELETION_FILE_LIST="dry-slave-deleted-list-$SYNC_ID" - else - DELETION_FILE_LIST="slave-deleted-list-$SYNC_ID" - fi - _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + _delete_local "$REPLICA_DIR" "slave$2" "$DELETE_DIR" & child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME retval=$? - if [ $retval == 0 ] + if [ $retval != 0 ] then - echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" - else LogError "Deletion on replica $1 failed." - echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" exit 1 fi else REPLICA_DIR="$SLAVE_SYNC_DIR" DELETE_DIR="$SLAVE_DELETE_DIR" - if [ $dryrun -eq 1 ] - then - DELETION_FILE_LIST="dry-master-deleted-list-$SYNC_ID" - else - DELETION_FILE_LIST="master-deleted-list-$SYNC_ID" - fi if [ "$REMOTE_SYNC" == "yes" ] then - _delete_remote "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + _delete_remote "$REPLICA_DIR" "master$2" "$DELETE_DIR" & else - _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + _delete_local "$REPLICA_DIR" "master$2" "$DELETE_DIR" & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME @@ -1268,27 +1220,33 @@ function deletion_propagation then Log "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" fi - echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" + return $retval else LogError "Deletion on remote system failed." if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] then LogError "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" fi - echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" - exit 1 + exit 1 fi fi } -###### Sync function in 10 steps (functions above) +###### Sync function in 5 steps of each 2 runs (functions above) +###### +###### Step 1: Create current tree list for master and slave replicas (Steps 1M and 1S) +###### Step 2: Create deleted file list for master and slave replicas (Steps 2M and 2S) +###### Step 3: Update master and slave replicas (Steps 3M and 3S, order depending on conflict prevalence) +###### Step 4: Deleted file propagation to master and slave replicas (Steps 4M and 4S) +###### Step 5: Create after run tree list for master and slave replicas (Steps 5M and 5S) + function Sync { Log "Starting synchronization task." CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - if [ -f "$MASTER_LAST_ACTION" ] && [ "$RESUME_SYNC" == "yes" ] + if [ -f "$MASTER_LAST_ACTION" ] && [ "$RESUME_SYNC" != "no" ] then resume_sync=$(cat "$MASTER_LAST_ACTION") if [ -f "$MASTER_RESUME_COUNT" ] @@ -1302,21 +1260,15 @@ function Sync then if [ "$resume_sync" != "sync.success" ] then - Log "WARNING: Trying to resume aborted osync execution on $(stat --format %y "$MASTER_LAST_ACTION") at task [$resume_sync]. [$resume_count] previous tries." - if [ $dryrun -ne 1 ] - then - echo $(($resume_count+1)) > "$MASTER_RESUME_COUNT" - fi + Log "WARNING: Trying to resume aborted osync execution on $($STAT_CMD "$MASTER_LAST_ACTION") at task [$resume_sync]. [$resume_count] previous tries." + echo $(($resume_count+1)) > "$MASTER_RESUME_COUNT" else resume_sync=none fi else Log "Will not resume aborted osync execution. Too much resume tries [$resume_count]." - if [ $dryrun -ne 1 ] - then - echo "noresume" > "$MASTER_LAST_ACTION" - echo "0" > "$MASTER_RESUME_COUNT" - fi + echo "noresume" > "$MASTER_LAST_ACTION" + echo "0" > "$MASTER_RESUME_COUNT" resume_sync=none fi else @@ -1330,82 +1282,151 @@ function Sync if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "master-replica-tree.fail" ] then #master_tree_current - tree_list "$MASTER_SYNC_DIR" master-tree-current master-replica-tree + tree_list "$MASTER_SYNC_DIR" master "$TREE_CURRENT_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[0]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[0]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "master-replica-tree.success" ] || [ "$resume_sync" == "slave-replica-tree.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[0]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[1]}.fail" ] then #slave_tree_current - tree_list "$SLAVE_SYNC_DIR" slave-tree-current slave-replica-tree + tree_list "$SLAVE_SYNC_DIR" slave "$TREE_CURRENT_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[1]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[1]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-tree.success" ] || [ "$resume_sync" == "master-replica-deleted-list.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[1]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[2]}.fail" ] then - delete_list master + delete_list master "$TREE_AFTER_FILENAME" "$TREE_CURRENT_FILENAME" "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[2]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNc_ACTION[2]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "master-replica-deleted-list.success" ] || [ "$resume_sync" == "slave-replica-deleted-list.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[2]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.fail" ] then - delete_list slave + delete_list slave "$TREE_AFTER_FILENAME" "$TREE_CURRENT_FILENAME" "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[3]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[3]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-deleted-list.success" ] || [ "$resume_sync" == "update-master-replica.fail" ] || [ "$resume_sync" == "update-slave-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] then if [ "$CONFLICT_PREVALANCE" != "master" ] then - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-deleted-list.success" ] || [ "$resume_sync" == "update-master-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] then - sync_update slave master + sync_update slave master "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[4]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[4]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-master-replica.success" ] || [ "$resume_sync" == "update-slave-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] then - sync_update master slave + sync_update master slave "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[5]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[5]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi else - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-deleted-list.success" ] || [ "$resume_sync" == "update-slave-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] then - sync_update master slave + sync_update master slave "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[5]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[5]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-slave-replica.success" ] || [ "$resume_sync" == "update-master-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] then - sync_update slave master + sync_update slave master "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[4]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[4]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi fi fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-slave-replica.success" ] || [ "$resume_sync" == "update-master-replica.success" ] || [ "$resume_sync" == "delete-propagation-slave.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] then - deletion_propagation slave + deletion_propagation slave "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[6]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[6]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "delete-propagation-slave.success" ] || [ "$resume_sync" == "delete-propagation-master.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ] then - deletion_propagation master + deletion_propagation master "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[7]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[7]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "delete-propagation-master.success" ] || [ "$resume_sync" == "master-replica-tree-after.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ] then #master_tree_after - tree_list "$MASTER_SYNC_DIR" master-tree-after master-replica-tree-after + tree_list "$MASTER_SYNC_DIR" master "$TREE_AFTER_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[8]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[8]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "master-replica-tree-after.success" ] || [ "$resume_sync" == "slave-replica-tree-after.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ] then #slave_tree_after - tree_list "$SLAVE_SYNC_DIR" slave-tree-after slave-replica-tree-after + tree_list "$SLAVE_SYNC_DIR" slave "$TREE_AFTER_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[9]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[9]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi Log "Finished synchronization task." - echo "sync.success" > "$MASTER_LAST_ACTION" + echo "${SYNC_ACTION[10]}" > "$MASTER_LAST_ACTION" - if [ $dryrun -ne 1 ] - then - echo "0" > "$MASTER_RESUME_COUNT" - fi + echo "0" > "$MASTER_RESUME_COUNT" } function SoftDelete @@ -1599,21 +1620,6 @@ function Init MAIL_ALERT_MSG="Warning: Execution of osync instance $OSYNC_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors on $(date)." - ## If running Msys, find command of windows is used instead of msys one - if [ "$LOCAL_OS" == "msys" ] - then - FIND_CMD=$(dirname $BASH)/find - else - FIND_CMD=find - fi - - if [ "$REMOTE_OS" == "msys" ] - then - REMOTE_FIND_CMD=$(dirname $BASH)/find - else - REMOTE_FIND_CMD=find - fi - ## Test if slave dir is a ssh uri, and if yes, break it down it its values if [ "${SLAVE_SYNC_DIR:0:6}" == "ssh://" ] then @@ -1653,15 +1659,6 @@ function Init MASTER_LOCK="$MASTER_STATE_DIR/lock" SLAVE_LOCK="$SLAVE_STATE_DIR/lock" - if [ $dryrun -eq 1 ] - then - MASTER_LAST_ACTION="$MASTER_STATE_DIR/dry-last-action-$SYNC_ID" - MASTER_RESUME_COUNT="$MASTER_STATE_DIR/dry-resume-count-$SYNC_ID" - else - MASTER_LAST_ACTION="$MASTER_STATE_DIR/last-action-$SYNC_ID" - MASTER_RESUME_COUNT="$MASTER_STATE_DIR/resume-count-$SYNC_ID" - fi - ## Working directories to keep backups of updated / deleted files MASTER_BACKUP_DIR="$OSYNC_DIR/backups" MASTER_DELETE_DIR="$OSYNC_DIR/deleted" @@ -1713,11 +1710,6 @@ function Init ## Set rsync default arguments RSYNC_ARGS="-rlptgoD" - ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] - then - RSYNC_ARGS=$RSYNC_ARGS"E" - fi if [ "$PRESERVE_ACL" == "yes" ] then RSYNC_ARGS=$RSYNC_ARGS"A" @@ -1777,6 +1769,76 @@ function Init RsyncExcludePattern ## Add Rsync exclude from file RsyncExcludeFrom + + ## Filenames for state files + if [ $dryrun -eq 1 ] + then + dry_suffix="-dry" + fi + + TREE_CURRENT_FILENAME="-tree-current-$SYNC_ID$dry_suffix" + TREE_AFTER_FILENAME="-tree-after-$SYNC_ID$dry_suffix" + DELETED_LIST_FILENAME="-deleted-list$SYNC_ID$dry_suffix" + MASTER_LAST_ACTION="$MASTER_STATE_DIR/last-action-$SYNC_ID$dry_suffix" + MASTER_RESUME_COUNT="$MASTER_STATE_DIR/resume-count-$SYNC_ID$dry_suffix" + + ## Sync function actions (0-9) + SYNC_ACTION=( + 'master-replica-tree' + 'slave-replica-tree' + 'master-deleted-list' + 'slave-deleted-list' + 'update-master-replica' + 'update-slave-replica' + 'delete-propagation-slave' + 'delete-propagation-master' + 'master-replica-tree-after' + 'slave-replica-tree-after' + 'sync.success' + ) +} + +function InitLocalOSSettings +{ + ## If running Msys, find command of windows is used instead of msys one + if [ "$LOCAL_OS" == "msys" ] + then + FIND_CMD=$(dirname $BASH)/find + else + FIND_CMD=find + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ] + then + STAT_CMD="stat -f \"%Sm\"" + else + STAT_CMD="stat --format %y" + fi + + ## Ping command has different syntax on Msys and others + if [ "$LOCAL_OS" == "msys" ] + then + PING_CMD="ping -n 2" + else + PING_CMD="ping -c 2 -i .2" + fi +} + +function InitRemoteOSSettings +{ + ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] + then + RSYNC_ARGS=$RSYNC_ARGS"E" + fi + + if [ "$REMOTE_OS" == "msys" ] + then + REMOTE_FIND_CMD=$(dirname $BASH)/find + else + REMOTE_FIND_CMD=find + fi } function Main @@ -1933,6 +1995,8 @@ done # Remove leading space if there is one opts="${opts# *}" +GetLocalOS +InitLocalOSSettings CheckEnvironment if [ $? == 0 ] then @@ -1956,6 +2020,8 @@ then LoadConfigFile "$ConfigFile" fi Init + GetRemoteOS + InitRemoteOSSettings if [ $sync_on_changes -eq 1 ] then SyncOnChanges @@ -1965,7 +2031,6 @@ then Log "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." Log "-------------------------------------------------------------" Log "Sync task [$SYNC_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" - GetOperatingSystem if [ $no_maxtime -eq 1 ] then SOFT_MAX_EXEC_TIME=0 diff --git a/sync.conf b/sync.conf index 79d0247..5b97ada 100755 --- a/sync.conf +++ b/sync.conf @@ -1,8 +1,8 @@ -#!/bin/bash +#!/usr/bin/env bash ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2505201402 +###### Config file rev 2605201401 ## ---------- GENERAL OPTIONS From fa19036de633386a3a4671e943a1f7403e36a34b Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 27 May 2014 11:35:45 +0200 Subject: [PATCH 4/5] Added follow symlink option + other minor fixes --- CHANGELOG.md | 5 ++++- osync.sh | 18 +++++++++++------- sync.conf | 6 ++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e05180a..b3a9106 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,16 +14,19 @@ FAR FUTURE IMPROVEMENTS KNOWN ISSUES ------------ -- Still need more testing on BSD, MacOSX and Windows MSYS +- RC3 is tested against Linux, FreeBSD and MacOS X. More testing needed in MSYS Windows environment. - Cannot finish sync if one replica contains a directory and the other replica contains a file named the same way (Unix doesn't allow this) UNDER WORK ---------- +- sync test automation RECENT CHANGES -------------- +- Added follow symlink parameter +- Minor fixes in parameter list when bandwidth parameter is used - Added some additionnal checks for *BSD and MacOS environments - Changed /bin/bash to /usr/bin/env bash for sanity on other systems, also check for bash presence before running - Changed default behavior for quick sync tasks: Will try to resume failed sync tasks once diff --git a/osync.sh b/osync.sh index 52f79ee..ea80dbb 100755 --- a/osync.sh +++ b/osync.sh @@ -4,7 +4,7 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=0.99RC3 -PROGRAM_BUILD=2605201403 +PROGRAM_BUILD=2705201401 if ! type -p "$BASH" > /dev/null then @@ -1712,23 +1712,27 @@ function Init if [ "$PRESERVE_ACL" == "yes" ] then - RSYNC_ARGS=$RSYNC_ARGS"A" + RSYNC_ARGS=$RSYNC_ARGS" -A" fi if [ "$PRESERVE_XATTR" == "yes" ] then - RSYNC_ARGS=$RSYNC_ARGS"X" + RSYNC_ARGS=$RSYNC_ARGS" -X" fi if [ "$RSYNC_COMPRESS" == "yes" ] then - RSYNC_ARGS=$RSYNC_ARGS"z" + RSYNC_ARGS=$RSYNC_ARGS" -z" fi + if [ "$COPY_SYMLINKS" == "yes" ] + then + RSYNC_ARGS=$RSYNC_ARGS" -L" + fi if [ "$PRESERVE_HARDLINKS" == "yes" ] then - RSYNC_ARGS=$RSYNC_ARGS"H" + RSYNC_ARGS=$RSYNC_ARGS" -H" fi if [ $dryrun -eq 1 ] then - RSYNC_ARGS=$RSYNC_ARGS"n" + RSYNC_ARGS=$RSYNC_ARGS" -n" DRY_WARNING="/!\ DRY RUN" fi @@ -1830,7 +1834,7 @@ function InitRemoteOSSettings ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] then - RSYNC_ARGS=$RSYNC_ARGS"E" + RSYNC_ARGS=$RSYNC_ARGS" -E" fi if [ "$REMOTE_OS" == "msys" ] diff --git a/sync.conf b/sync.conf index 5b97ada..36b03da 100755 --- a/sync.conf +++ b/sync.conf @@ -2,7 +2,7 @@ ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2605201401 +###### Config file rev 2705201401 ## ---------- GENERAL OPTIONS @@ -67,6 +67,8 @@ RSYNC_REMOTE_PATH="" PRESERVE_ACL=no ## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors. PRESERVE_XATTR=no +## Transforms symlinks into referent files/dirs +COPY_SYMLINKS=no ## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them. PRESERVE_HARDLINKS=no @@ -80,7 +82,7 @@ HARD_MAX_EXEC_TIME=10600 ## Minimum time (in seconds) in file monitor mode between modification detection and sync task in order to let copy operations finish. MIN_WAIT=60 -## ---------- BACKUP AND TRASH OPTIONS +## ---------- BACKUP AND DELETION OPTIONS ## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups CONFLICT_BACKUP=yes From eeb0ef372291999699f69f81168831175aac0741 Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 27 May 2014 12:47:50 +0200 Subject: [PATCH 5/5] Fixes for RC3 --- CHANGELOG.md | 3 +++ osync.sh | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3a9106..4caa046 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ UNDER WORK RECENT CHANGES -------------- +- 27 May 2014: Osync 0.99 RC3 +- Additionnal delete fix for *BSD and MSYS (deleted file list not created right) +- Fixed dry mode to use non dry after run treelists to create delete lists - Added follow symlink parameter - Minor fixes in parameter list when bandwidth parameter is used - Added some additionnal checks for *BSD and MacOS environments diff --git a/osync.sh b/osync.sh index ea80dbb..10fda1e 100755 --- a/osync.sh +++ b/osync.sh @@ -4,8 +4,9 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=0.99RC3 -PROGRAM_BUILD=2705201401 +PROGRAM_BUILD=2705201403 +## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode if ! type -p "$BASH" > /dev/null then echo "Please run this script only with bash shell. Tested on bash >= 3.2" @@ -950,15 +951,15 @@ function tree_list function delete_list { Log "Creating $1 replica deleted file list." - if [ -f "$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID" ] + if [ -f "$MASTER_STATE_DIR/$1$TREE_AFTER_FILENAME_NO_SUFFIX" ] then ## Same functionnality, comm is much faster than grep but is not available on every platform if type -p comm > /dev/null 2>&1 then - cmd="comm -23 \"$MASTER_STATE_DIR/$1$2\" \"$MASTER_STATE_DIR/$1$3\" > \"$MASTER_STATE_DIR/$1$4\"" + cmd="comm -23 \"$MASTER_STATE_DIR/$1$TREE_AFTER_FILENAME_NO_SUFFIX\" \"$MASTER_STATE_DIR/$1$3\" > \"$MASTER_STATE_DIR/$1$4\"" else ## The || : forces the command to have a good result - cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/$1$3\" \"$MASTER_STATE_DIR/$1$2\" || : > \"$MASTER_STATE_DIR/$1$4\"" + cmd="(grep -F -x -v -f \"$MASTER_STATE_DIR/$1$3\" \"$MASTER_STATE_DIR/$1$TREE_AFTER_FILENAME_NO_SUFFIX\" || :) > \"$MASTER_STATE_DIR/$1$4\"" fi LogDebug "CMD: $cmd" @@ -1782,6 +1783,7 @@ function Init TREE_CURRENT_FILENAME="-tree-current-$SYNC_ID$dry_suffix" TREE_AFTER_FILENAME="-tree-after-$SYNC_ID$dry_suffix" + TREE_AFTER_FILENAME_NO_SUFFIX="-tree-after-$SYNC_ID" DELETED_LIST_FILENAME="-deleted-list$SYNC_ID$dry_suffix" MASTER_LAST_ACTION="$MASTER_STATE_DIR/last-action-$SYNC_ID$dry_suffix" MASTER_RESUME_COUNT="$MASTER_STATE_DIR/resume-count-$SYNC_ID$dry_suffix"