Code cleanup, preparing sudo rsync and rsync executable support.
This commit is contained in:
parent
ea36ae0f1b
commit
1f9b84dae3
16
CHANGELOG.md
16
CHANGELOG.md
|
@ -1,7 +1,17 @@
|
|||
! Tag as v1.0RC1
|
||||
! verify spaces in sync dirs
|
||||
! Support for ACL and Xattr
|
||||
TODO LIST
|
||||
---------
|
||||
|
||||
! Tag as v1.0 beta
|
||||
! verify spaces in sync dir names
|
||||
! sudo support for rsync and generic cmds
|
||||
! add osync support in ssh_filter.sh
|
||||
! minimum space checks
|
||||
! different rsync executable
|
||||
|
||||
RECENT CHANGES
|
||||
--------------
|
||||
|
||||
- Added support for ACL and xattr
|
||||
- Added --force-unlock parameter to bypass any existing locks on replicas
|
||||
- Added full remote support for slave replica
|
||||
- Improved error detection
|
||||
|
|
88
osync.sh
88
osync.sh
|
@ -4,7 +4,7 @@
|
|||
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
||||
|
||||
OSYNC_VERSION=0.8
|
||||
OSYNC_BUILD=2107201304
|
||||
OSYNC_BUILD=2207201301
|
||||
|
||||
DEBUG=no
|
||||
SCRIPT_PID=$$
|
||||
|
@ -380,24 +380,6 @@ function RunAfterHook
|
|||
fi
|
||||
}
|
||||
|
||||
function SetSudoOptions
|
||||
{
|
||||
## Add this to support prior config files without RSYNC_EXECUTABLE option
|
||||
if [ "$RSYNC_EXECUTABLE" == "" ]
|
||||
then
|
||||
RSYNC_EXECUTABLE=rsync
|
||||
fi
|
||||
|
||||
if [ "$SUDO_EXEC" == "yes" ]
|
||||
then
|
||||
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
|
||||
COMMAND_SUDO="sudo"
|
||||
else
|
||||
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
|
||||
COMMAND_SUDO=""
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckConnectivityRemoteHost
|
||||
{
|
||||
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ]
|
||||
|
@ -623,13 +605,13 @@ function LockDirectories
|
|||
then
|
||||
if [ "$slave_lock_id" == "$SYNC_ID" ]
|
||||
then
|
||||
Log "There is a dead osync lock on slave replica that corresponds to this master. Instance $slave_lock_pid no longer running. Resuming."
|
||||
Log "There is a dead osync lock on slave replica that corresponds to this master sync-id. Instance $slave_lock_pid no longer running. Resuming."
|
||||
else
|
||||
if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]
|
||||
then
|
||||
LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master. Forcing resume."
|
||||
LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master sync-id. Forcing resume."
|
||||
else
|
||||
LogError "There is a dead osync lock on slave replica that does not correspond to this master. Will not resume."
|
||||
LogError "There is a dead osync lock on slave replica that does not correspond to this master sync-id. Will not resume."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
@ -676,7 +658,8 @@ function UnlockDirectories
|
|||
function master_tree_current
|
||||
{
|
||||
Log "Creating master replica file list."
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &
|
||||
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgodE --exclude \"$OSYNC_DIR\" --list-only \"$MASTER_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{print $5}' | (grep -v \"^\.$\" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &"
|
||||
eval $rsync_cmd
|
||||
child_pid=$!
|
||||
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
|
||||
if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ]
|
||||
|
@ -695,9 +678,9 @@ function slave_tree_current
|
|||
Log "Creating slave replica file list."
|
||||
if [ "$REMOTE_SYNC" == "yes" ]
|
||||
then
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
|
||||
$(which $RSYNC_EXECUTABLE) -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
|
||||
else
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
|
||||
$(which $RSYNC_EXECUTABLe) -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
|
||||
|
@ -744,9 +727,9 @@ function sync_update_slave
|
|||
Log "Updating slave replica."
|
||||
if [ "$REMOTE_SYNC" == "yes" ]
|
||||
then
|
||||
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
|
||||
else
|
||||
rsync $DRY_OPTION -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
|
||||
|
@ -772,9 +755,9 @@ function sync_update_master
|
|||
Log "Updating master replica."
|
||||
if [ "$REMOTE_SYNC" == "yes" ]
|
||||
then
|
||||
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
|
||||
else
|
||||
rsync $DRY_OPTION -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
|
||||
|
@ -800,9 +783,9 @@ function delete_on_slave
|
|||
Log "Propagating deletitions to slave replica."
|
||||
if [ "$REMOTE_SYNC" == "yes" ]
|
||||
then
|
||||
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
|
||||
else
|
||||
rsync $DRY_OPTION -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
|
||||
|
@ -827,9 +810,9 @@ function delete_on_master
|
|||
Log "Propagating deletitions to master replica."
|
||||
if [ "$REMOTE_SYNC" == "yes" ]
|
||||
then
|
||||
rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
|
||||
else
|
||||
rsync $DRY_OPTION -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
|
||||
rsync $RSYNC_ARGS -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
|
||||
|
@ -1070,13 +1053,44 @@ function Init
|
|||
RSYNC_SSH_CMD="$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT"
|
||||
fi
|
||||
|
||||
## Dryrun option
|
||||
## Set rsync executable and rsync path (for remote sudo rsync)
|
||||
if [ "$RSYNC_EXECUTABLE" == "" ]
|
||||
then
|
||||
RSYNC_EXECUTABLE=rsync
|
||||
fi
|
||||
|
||||
if [ "$SUDO_EXEC" == "yes" ]
|
||||
then
|
||||
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
|
||||
COMMAND_SUDO="sudo"
|
||||
else
|
||||
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
|
||||
COMMAND_SUDO=""
|
||||
fi
|
||||
|
||||
|
||||
## Set rsync options
|
||||
RSYNC_ARGS="-"
|
||||
if [ "$PRESERVE_ACLS" == "yes" ]
|
||||
then
|
||||
RSYNC_ARGS=$RSYNC_ARGS"A"
|
||||
fi
|
||||
if [ "$PRESERVE_XATTR" == "yes" ]
|
||||
then
|
||||
RSYNC_ARGS=$RSYNC_ARGS"X"
|
||||
fi
|
||||
if [ "$RSYNC_COMPRESS" == "yes" ]
|
||||
then
|
||||
RSYNC_ARGS=$RSYNC_ARGS"z"
|
||||
fi
|
||||
if [ $dryrun -eq 1 ]
|
||||
then
|
||||
DRY_OPTION=--dry-run
|
||||
RSYNC_ARGS=$RSYNC_ARGS"n"
|
||||
DRY_WARNING="/!\ DRY RUN"
|
||||
else
|
||||
DRY_OPTION=
|
||||
fi
|
||||
if [ "$RSYNC_ARGS" == "-" ]
|
||||
then
|
||||
RSYNC_ARGS=""
|
||||
fi
|
||||
|
||||
## Conflict options
|
||||
|
|
34
sync.conf
34
sync.conf
|
@ -2,16 +2,16 @@
|
|||
|
||||
###### Osync - Rsync based two way sync engine with fault tolerance
|
||||
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
||||
#### Config file rev 2107201302
|
||||
#### Config file rev 2207201301
|
||||
|
||||
## Sync job identification, any string you want
|
||||
## Sync job identification, any string you want, no spaces
|
||||
SYNC_ID="sync_test"
|
||||
|
||||
## Directories to synchronize
|
||||
MASTER_SYNC_DIR="/home/git/osync/test/dir1"
|
||||
SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
|
||||
|
||||
## Create directories if they do not exist
|
||||
## Create sync directories if they do not exist
|
||||
CREATE_DIRS=yes
|
||||
|
||||
## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default).
|
||||
|
@ -33,37 +33,57 @@ SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
|
|||
REMOTE_USER=backupmaster
|
||||
REMOTE_HOST=badministrateur.com
|
||||
REMOTE_PORT=48884
|
||||
## ssh compression should be used unless your remote connection is good enough (LAN)
|
||||
SSH_COMPRESSION=yes
|
||||
## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
|
||||
REMOTE_HOST_PING=no
|
||||
## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
|
||||
REMOTE_3RD_PARTY_HOST="www.kernel.org"
|
||||
|
||||
## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors.
|
||||
PRESERVE_ACL=yes
|
||||
## Preserve Xattr
|
||||
PRESERVE_XATTR=yes
|
||||
## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression.
|
||||
RSYNC_COMPRESS=yes
|
||||
|
||||
## Maximum execution time for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process.
|
||||
SOFT_MAX_EXEC_TIME=30000
|
||||
HARD_MAX_EXEC_TIME=36000
|
||||
|
||||
## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner
|
||||
CONFLICT_PREVALANCE=master
|
||||
## Keep a backup of a file if gets updated from remote side
|
||||
CONFLICT_BACKUP=yes
|
||||
CONFLICT_BACKUP_DAYS=30
|
||||
# This can be very space consuming
|
||||
## Keep multiple backups of a file if it gets updated from remote side. This can be very space consuming
|
||||
CONFLICT_BACKUP_MULTIPLE=yes
|
||||
## Number of days to keep backups
|
||||
CONFLICT_BACKUP_DAYS=30
|
||||
|
||||
## On deletition propagation to sync partner, keep a backup of deleted files on sync partner
|
||||
SOFT_DELETE=yes
|
||||
## Number of days to keep deleted files
|
||||
SOFT_DELETE_DAYS=30
|
||||
|
||||
RESUME_SYNC=no
|
||||
## Resume an aborted sync task
|
||||
RESUME_SYNC=yes
|
||||
## Number of times to try resuming before initating a new sync
|
||||
RESUME_TRY=2
|
||||
FORCE_STRANGER_LOCK_RESUME=yes
|
||||
## When a dead pidlock exists on slave that does not correspond to master's sync-id, force pidlock removal
|
||||
FORCE_STRANGER_LOCK_RESUME=no
|
||||
|
||||
## List of alert mails separated by spaces
|
||||
DESTINATION_MAILS="ozy@badministrateur.com"
|
||||
|
||||
## Run local commands before and after sync task
|
||||
LOCAL_RUN_BEFORE_CMD=""
|
||||
LOCAL_RUN_AFTER_CMD=""
|
||||
|
||||
## Run commands on remote slave befre and after sync task
|
||||
REMOTE_RUN_BEFORE_CMD="du /var/log"
|
||||
REMOTE_RUN_AFTER_CMD="du /tmp"
|
||||
|
||||
## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIMEe. Set this to 0 to disable killing.
|
||||
MAX_EXEC_TIME_PER_CMD_BEFORE=0
|
||||
## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command.
|
||||
MAX_EXEC_TIME_PER_CMD_AFTER=0
|
||||
|
|
Loading…
Reference in New Issue