WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Code cleanup, preparing sudo rsync and rsync executable support.

This commit is contained in:
deajan 2013-07-22 11:34:58 +02:00
parent ea36ae0f1b
commit 1f9b84dae3
3 changed files with 91 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG.md
View File

@ -1,7 +1,17 @@
! Tag as v1.0RC1 TODO LIST
! verify spaces in sync dirs ---------
! Support for ACL and Xattr
! Tag as v1.0 beta
! verify spaces in sync dir names
! sudo support for rsync and generic cmds
! add osync support in ssh_filter.sh
! minimum space checks
! different rsync executable
RECENT CHANGES
--------------
- Added support for ACL and xattr
- Added --force-unlock parameter to bypass any existing locks on replicas - Added --force-unlock parameter to bypass any existing locks on replicas
- Added full remote support for slave replica - Added full remote support for slave replica
- Improved error detection - Improved error detection

88
osync.sh
View File

@ -4,7 +4,7 @@
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
OSYNC_VERSION=0.8 OSYNC_VERSION=0.8
OSYNC_BUILD=2107201304 OSYNC_BUILD=2207201301
DEBUG=no DEBUG=no
SCRIPT_PID=$$ SCRIPT_PID=$$
@ -380,24 +380,6 @@ function RunAfterHook
fi fi
} }
function SetSudoOptions
{
## Add this to support prior config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]
then
RSYNC_EXECUTABLE=rsync
fi
if [ "$SUDO_EXEC" == "yes" ]
then
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO="sudo"
else
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO=""
fi
}
function CheckConnectivityRemoteHost function CheckConnectivityRemoteHost
{ {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ] if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ]
@ -623,13 +605,13 @@ function LockDirectories
then then
if [ "$slave_lock_id" == "$SYNC_ID" ] if [ "$slave_lock_id" == "$SYNC_ID" ]
then then
Log "There is a dead osync lock on slave replica that corresponds to this master. Instance $slave_lock_pid no longer running. Resuming." Log "There is a dead osync lock on slave replica that corresponds to this master sync-id. Instance $slave_lock_pid no longer running. Resuming."
else else
if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ] if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]
then then
LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master. Forcing resume." LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master sync-id. Forcing resume."
else else
LogError "There is a dead osync lock on slave replica that does not correspond to this master. Will not resume." LogError "There is a dead osync lock on slave replica that does not correspond to this master sync-id. Will not resume."
exit 1 exit 1
fi fi
fi fi
@ -676,7 +658,8 @@ function UnlockDirectories
function master_tree_current function master_tree_current
{ {
Log "Creating master replica file list." Log "Creating master replica file list."
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID & rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgodE --exclude \"$OSYNC_DIR\" --list-only \"$MASTER_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{print $5}' | (grep -v \"^\.$\" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &"
eval $rsync_cmd
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ] if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ]
@ -695,9 +678,9 @@ function slave_tree_current
Log "Creating slave replica file list." Log "Creating slave replica file list."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & $(which $RSYNC_EXECUTABLE) -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
else else
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & $(which $RSYNC_EXECUTABLe) -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -744,9 +727,9 @@ function sync_update_slave
Log "Updating slave replica." Log "Updating slave replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
else else
rsync $DRY_OPTION -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -772,9 +755,9 @@ function sync_update_master
Log "Updating master replica." Log "Updating master replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
else else
rsync $DRY_OPTION -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -800,9 +783,9 @@ function delete_on_slave
Log "Propagating deletitions to slave replica." Log "Propagating deletitions to slave replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
else else
rsync $DRY_OPTION -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
@ -827,9 +810,9 @@ function delete_on_master
Log "Propagating deletitions to master replica." Log "Propagating deletitions to master replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
else else
rsync $DRY_OPTION -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & rsync $RSYNC_ARGS -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
@ -1070,13 +1053,44 @@ function Init
RSYNC_SSH_CMD="$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" RSYNC_SSH_CMD="$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT"
fi fi
## Dryrun option ## Set rsync executable and rsync path (for remote sudo rsync)
if [ "$RSYNC_EXECUTABLE" == "" ]
then
RSYNC_EXECUTABLE=rsync
fi
if [ "$SUDO_EXEC" == "yes" ]
then
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO="sudo"
else
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO=""
fi
## Set rsync options
RSYNC_ARGS="-"
if [ "$PRESERVE_ACLS" == "yes" ]
then
RSYNC_ARGS=$RSYNC_ARGS"A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]
then
RSYNC_ARGS=$RSYNC_ARGS"X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]
then
RSYNC_ARGS=$RSYNC_ARGS"z"
fi
if [ $dryrun -eq 1 ] if [ $dryrun -eq 1 ]
then then
DRY_OPTION=--dry-run RSYNC_ARGS=$RSYNC_ARGS"n"
DRY_WARNING="/!\ DRY RUN" DRY_WARNING="/!\ DRY RUN"
else fi
DRY_OPTION= if [ "$RSYNC_ARGS" == "-" ]
then
RSYNC_ARGS=""
fi fi
## Conflict options ## Conflict options

34
sync.conf
View File

@ -2,16 +2,16 @@
###### Osync - Rsync based two way sync engine with fault tolerance ###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
#### Config file rev 2107201302 #### Config file rev 2207201301
## Sync job identification, any string you want ## Sync job identification, any string you want, no spaces
SYNC_ID="sync_test" SYNC_ID="sync_test"
## Directories to synchronize ## Directories to synchronize
MASTER_SYNC_DIR="/home/git/osync/test/dir1" MASTER_SYNC_DIR="/home/git/osync/test/dir1"
SLAVE_SYNC_DIR="/home/git/osync/test/dir2" SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
## Create directories if they do not exist ## Create sync directories if they do not exist
CREATE_DIRS=yes CREATE_DIRS=yes
## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default). ## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default).
@ -33,37 +33,57 @@ SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
REMOTE_USER=backupmaster REMOTE_USER=backupmaster
REMOTE_HOST=badministrateur.com REMOTE_HOST=badministrateur.com
REMOTE_PORT=48884 REMOTE_PORT=48884
## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes SSH_COMPRESSION=yes
## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
REMOTE_HOST_PING=no REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
REMOTE_3RD_PARTY_HOST="www.kernel.org" REMOTE_3RD_PARTY_HOST="www.kernel.org"
## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors.
PRESERVE_ACL=yes PRESERVE_ACL=yes
## Preserve Xattr
PRESERVE_XATTR=yes PRESERVE_XATTR=yes
## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression.
RSYNC_COMPRESS=yes RSYNC_COMPRESS=yes
## Maximum execution time for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process.
SOFT_MAX_EXEC_TIME=30000 SOFT_MAX_EXEC_TIME=30000
HARD_MAX_EXEC_TIME=36000 HARD_MAX_EXEC_TIME=36000
## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner
CONFLICT_PREVALANCE=master CONFLICT_PREVALANCE=master
## Keep a backup of a file if gets updated from remote side
CONFLICT_BACKUP=yes CONFLICT_BACKUP=yes
CONFLICT_BACKUP_DAYS=30 ## Keep multiple backups of a file if it gets updated from remote side. This can be very space consuming
# This can be very space consuming
CONFLICT_BACKUP_MULTIPLE=yes CONFLICT_BACKUP_MULTIPLE=yes
## Number of days to keep backups
CONFLICT_BACKUP_DAYS=30
## On deletition propagation to sync partner, keep a backup of deleted files on sync partner
SOFT_DELETE=yes SOFT_DELETE=yes
## Number of days to keep deleted files
SOFT_DELETE_DAYS=30 SOFT_DELETE_DAYS=30
RESUME_SYNC=no ## Resume an aborted sync task
RESUME_SYNC=yes
## Number of times to try resuming before initating a new sync
RESUME_TRY=2 RESUME_TRY=2
FORCE_STRANGER_LOCK_RESUME=yes ## When a dead pidlock exists on slave that does not correspond to master's sync-id, force pidlock removal
FORCE_STRANGER_LOCK_RESUME=no
## List of alert mails separated by spaces
DESTINATION_MAILS="ozy@badministrateur.com" DESTINATION_MAILS="ozy@badministrateur.com"
## Run local commands before and after sync task
LOCAL_RUN_BEFORE_CMD="" LOCAL_RUN_BEFORE_CMD=""
LOCAL_RUN_AFTER_CMD="" LOCAL_RUN_AFTER_CMD=""
## Run commands on remote slave befre and after sync task
REMOTE_RUN_BEFORE_CMD="du /var/log" REMOTE_RUN_BEFORE_CMD="du /var/log"
REMOTE_RUN_AFTER_CMD="du /tmp" REMOTE_RUN_AFTER_CMD="du /tmp"
## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIMEe. Set this to 0 to disable killing.
MAX_EXEC_TIME_PER_CMD_BEFORE=0 MAX_EXEC_TIME_PER_CMD_BEFORE=0
## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command.
MAX_EXEC_TIME_PER_CMD_AFTER=0 MAX_EXEC_TIME_PER_CMD_AFTER=0