WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Code cleanup, preparing sudo rsync and rsync executable support.

This commit is contained in:
deajan 2013-07-22 11:34:58 +02:00
parent ea36ae0f1b
commit 1f9b84dae3
3 changed files with 91 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG.md
View File

@ -1,7 +1,17 @@
! Tag as v1.0RC1
! verify spaces in sync dirs
! Support for ACL and Xattr
TODO LIST
---------
! Tag as v1.0 beta
! verify spaces in sync dir names
! sudo support for rsync and generic cmds
! add osync support in ssh_filter.sh
! minimum space checks
! different rsync executable
RECENT CHANGES
--------------
- Added support for ACL and xattr
- Added --force-unlock parameter to bypass any existing locks on replicas
- Added full remote support for slave replica
- Improved error detection

88
osync.sh
View File

@ -4,7 +4,7 @@
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
OSYNC_VERSION=0.8
OSYNC_BUILD=2107201304
OSYNC_BUILD=2207201301
DEBUG=no
SCRIPT_PID=$$
@ -380,24 +380,6 @@ function RunAfterHook
fi
}
function SetSudoOptions
{
## Add this to support prior config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]
then
RSYNC_EXECUTABLE=rsync
fi
if [ "$SUDO_EXEC" == "yes" ]
then
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO="sudo"
else
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO=""
fi
}
function CheckConnectivityRemoteHost
{
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ]
@ -623,13 +605,13 @@ function LockDirectories
then
if [ "$slave_lock_id" == "$SYNC_ID" ]
then
Log "There is a dead osync lock on slave replica that corresponds to this master. Instance $slave_lock_pid no longer running. Resuming."
Log "There is a dead osync lock on slave replica that corresponds to this master sync-id. Instance $slave_lock_pid no longer running. Resuming."
else
if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]
then
LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master. Forcing resume."
LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master sync-id. Forcing resume."
else
LogError "There is a dead osync lock on slave replica that does not correspond to this master. Will not resume."
LogError "There is a dead osync lock on slave replica that does not correspond to this master sync-id. Will not resume."
exit 1
fi
fi
@ -676,7 +658,8 @@ function UnlockDirectories
function master_tree_current
{
Log "Creating master replica file list."
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgodE --exclude \"$OSYNC_DIR\" --list-only \"$MASTER_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{print $5}' | (grep -v \"^\.$\" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &"
eval $rsync_cmd
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ]
@ -695,9 +678,9 @@ function slave_tree_current
Log "Creating slave replica file list."
if [ "$REMOTE_SYNC" == "yes" ]
then
rsync -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
$(which $RSYNC_EXECUTABLE) -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
else
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
$(which $RSYNC_EXECUTABLe) -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -744,9 +727,9 @@ function sync_update_slave
Log "Updating slave replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
else
rsync $DRY_OPTION -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -772,9 +755,9 @@ function sync_update_master
Log "Updating master replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
else
rsync $DRY_OPTION -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -800,9 +783,9 @@ function delete_on_slave
Log "Propagating deletitions to slave replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
else
rsync $DRY_OPTION -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
@ -827,9 +810,9 @@ function delete_on_master
Log "Propagating deletitions to master replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
else
rsync $DRY_OPTION -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
rsync $RSYNC_ARGS -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
@ -1070,13 +1053,44 @@ function Init
RSYNC_SSH_CMD="$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT"
fi
## Dryrun option
## Set rsync executable and rsync path (for remote sudo rsync)
if [ "$RSYNC_EXECUTABLE" == "" ]
then
RSYNC_EXECUTABLE=rsync
fi
if [ "$SUDO_EXEC" == "yes" ]
then
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO="sudo"
else
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
COMMAND_SUDO=""
fi
## Set rsync options
RSYNC_ARGS="-"
if [ "$PRESERVE_ACLS" == "yes" ]
then
RSYNC_ARGS=$RSYNC_ARGS"A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]
then
RSYNC_ARGS=$RSYNC_ARGS"X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]
then
RSYNC_ARGS=$RSYNC_ARGS"z"
fi
if [ $dryrun -eq 1 ]
then
DRY_OPTION=--dry-run
RSYNC_ARGS=$RSYNC_ARGS"n"
DRY_WARNING="/!\ DRY RUN"
else
DRY_OPTION=
fi
if [ "$RSYNC_ARGS" == "-" ]
then
RSYNC_ARGS=""
fi
## Conflict options

34
sync.conf
View File

@ -2,16 +2,16 @@
###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
#### Config file rev 2107201302
#### Config file rev 2207201301
## Sync job identification, any string you want
## Sync job identification, any string you want, no spaces
SYNC_ID="sync_test"
## Directories to synchronize
MASTER_SYNC_DIR="/home/git/osync/test/dir1"
SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
## Create directories if they do not exist
## Create sync directories if they do not exist
CREATE_DIRS=yes
## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default).
@ -33,37 +33,57 @@ SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
REMOTE_USER=backupmaster
REMOTE_HOST=badministrateur.com
REMOTE_PORT=48884
## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes
## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
REMOTE_3RD_PARTY_HOST="www.kernel.org"
## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors.
PRESERVE_ACL=yes
## Preserve Xattr
PRESERVE_XATTR=yes
## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression.
RSYNC_COMPRESS=yes
## Maximum execution time for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process.
SOFT_MAX_EXEC_TIME=30000
HARD_MAX_EXEC_TIME=36000
## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner
CONFLICT_PREVALANCE=master
## Keep a backup of a file if gets updated from remote side
CONFLICT_BACKUP=yes
CONFLICT_BACKUP_DAYS=30
# This can be very space consuming
## Keep multiple backups of a file if it gets updated from remote side. This can be very space consuming
CONFLICT_BACKUP_MULTIPLE=yes
## Number of days to keep backups
CONFLICT_BACKUP_DAYS=30
## On deletition propagation to sync partner, keep a backup of deleted files on sync partner
SOFT_DELETE=yes
## Number of days to keep deleted files
SOFT_DELETE_DAYS=30
RESUME_SYNC=no
## Resume an aborted sync task
RESUME_SYNC=yes
## Number of times to try resuming before initating a new sync
RESUME_TRY=2
FORCE_STRANGER_LOCK_RESUME=yes
## When a dead pidlock exists on slave that does not correspond to master's sync-id, force pidlock removal
FORCE_STRANGER_LOCK_RESUME=no
## List of alert mails separated by spaces
DESTINATION_MAILS="ozy@badministrateur.com"
## Run local commands before and after sync task
LOCAL_RUN_BEFORE_CMD=""
LOCAL_RUN_AFTER_CMD=""
## Run commands on remote slave befre and after sync task
REMOTE_RUN_BEFORE_CMD="du /var/log"
REMOTE_RUN_AFTER_CMD="du /tmp"
## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIMEe. Set this to 0 to disable killing.
MAX_EXEC_TIME_PER_CMD_BEFORE=0
## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command.
MAX_EXEC_TIME_PER_CMD_AFTER=0