Patch merges plus bug fixes
This commit is contained in:
deajan
parent
deb0058b6e
commit
b65c5a22ff
12
CHANGELOG.md
12
CHANGELOG.md
|
|
@ -16,9 +16,21 @@ KNOWN ISSUES
|
|||
|
||||
- Still need more testing on BSD, MacOSX and Windows MSYS
|
||||
|
||||
UNDER WORK
|
||||
----------
|
||||
|
||||
!- Better deletion propagation (again). Using rsync for deletion propagation is definitly not working in all cases (especially empty sub directories)
|
||||
!- Mutlislave asynchronous sync support
|
||||
|
||||
RECENT CHANGES
|
||||
--------------
|
||||
|
||||
- Added an easier debug setting i.e DEBUG=yes ./osync.sh (Again, thanks to Ulrich Norbisrath)
|
||||
- Added hardlink preservation (Thanks to Ulrich Norbisrath)
|
||||
- Added external exclusion file support (Thanks to Pierre Clement)
|
||||
- Fixed some typos in doc and program itself (Thanks to Pierre Clement)
|
||||
- More detailled verbose status messages
|
||||
- More detailled status messages
|
||||
- Fixed a bug preventing propagation of empty directory deletions
|
||||
- Fixed a nasty bug preventing writing lock files on remote system as superuser
|
||||
- Gzipped logs are now deleted once sent
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
.bash_history
|
||||
.ssh
|
||||
.config
|
||||
.localized
|
||||
.AppleDouble/
|
||||
._*
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
System Volume Information
|
||||
$Recycle.Bin
|
||||
|
||||
55
osync.sh
55
osync.sh
|
|
@ -4,9 +4,14 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance
|
|||
AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong"
|
||||
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
|
||||
PROGRAM_VERSION=0.99preRC3
|
||||
PROGRAM_BUILD=3103201402
|
||||
PROGRAM_BUILD=0805201403
|
||||
|
||||
## allow debugging from command line with preceding ocsync with DEBUG=yes
|
||||
if [ ! "$DEBUG" == "yes" ]
|
||||
then
|
||||
DEBUG=no
|
||||
fi
|
||||
|
||||
DEBUG=no
|
||||
SCRIPT_PID=$$
|
||||
|
||||
LOCAL_USER=$(whoami)
|
||||
|
|
@ -735,6 +740,14 @@ function RsyncExcludePattern
|
|||
IFS=$OLD_IFS
|
||||
}
|
||||
|
||||
function RsyncExcludeFrom
|
||||
{
|
||||
if [ ! $RSYNC_EXCLUDE_FROM == "" ] && [ -e $RSYNC_EXCLUDE_FROM ]
|
||||
then
|
||||
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\""
|
||||
fi
|
||||
}
|
||||
|
||||
function WriteLockFiles
|
||||
{
|
||||
echo $SCRIPT_PID > "$MASTER_LOCK"
|
||||
|
|
@ -909,7 +922,7 @@ function tree_list
|
|||
ESC=$(EscapeSpaces "$1")
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$ESC/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &"
|
||||
else
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSNYC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &"
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &"
|
||||
fi
|
||||
LogDebug "RSYNC_CMD: $rsync_cmd"
|
||||
## Redirect commands stderr here to get rsync stderr output in logfile
|
||||
|
|
@ -1051,12 +1064,12 @@ function deletion_propagation
|
|||
if [ "$1" == "master" ]
|
||||
then
|
||||
#rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"-! */\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
else
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
fi
|
||||
else
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
|
||||
fi
|
||||
LogDebug "RSYNC_CMD: $rsync_cmd"
|
||||
eval "$rsync_cmd"
|
||||
|
|
@ -1127,7 +1140,7 @@ function Sync
|
|||
|
||||
################################################################################################################################################# Actual sync begins here
|
||||
|
||||
## This replaces the case statement below because ;& operator is not supported in bash 3.2... Code is more messy than case :(
|
||||
## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :(
|
||||
if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "master-replica-tree.fail" ]
|
||||
then
|
||||
#master_tree_current
|
||||
|
|
@ -1218,7 +1231,12 @@ function SoftDelete
|
|||
if [ $dryrun -eq 1 ]
|
||||
then
|
||||
Log "Listing backups older than $CONFLICT_BACKUP_DAYS days on master replica. Won't remove anything."
|
||||
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS &
|
||||
if [ $verbose -eq 1 ]
|
||||
then
|
||||
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS &
|
||||
else
|
||||
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS > /dev/null &
|
||||
fi
|
||||
else
|
||||
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica."
|
||||
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; &
|
||||
|
|
@ -1287,7 +1305,7 @@ function SoftDelete
|
|||
|
||||
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
|
||||
then
|
||||
if [ -w "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ]
|
||||
if [ -d "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ]
|
||||
then
|
||||
if [ $dryrun -eq 1 ]
|
||||
then
|
||||
|
|
@ -1517,6 +1535,10 @@ function Init
|
|||
then
|
||||
RSYNC_ARGS=$RSYNC_ARGS"z"
|
||||
fi
|
||||
if [ "$PRESERVE_HARDLINKS" != "no" ]
|
||||
then
|
||||
RSYNC_ARGS$RSYNC_ARGS"H"
|
||||
fi
|
||||
if [ $dryrun -eq 1 ]
|
||||
then
|
||||
RSYNC_ARGS=$RSYNC_ARGS"n"
|
||||
|
|
@ -1559,6 +1581,8 @@ function Init
|
|||
|
||||
## Add Rsync exclude patterns
|
||||
RsyncExcludePattern
|
||||
## Add Rsync exclude from file
|
||||
RsyncExcludeFrom
|
||||
}
|
||||
|
||||
function Main
|
||||
|
|
@ -1592,6 +1616,7 @@ function Usage
|
|||
echo "--master=\"\" Master replica path. Will contain state and backup directory (is mandatory)."
|
||||
echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)."
|
||||
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica"
|
||||
echo "--sync-id=\"\" Optional task-id to identify this synchronization task when using multiple slaves."
|
||||
exit 128
|
||||
}
|
||||
|
||||
|
|
@ -1693,6 +1718,10 @@ do
|
|||
SSH_RSA_PRIVATE_KEY=${i##*=}
|
||||
opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\""
|
||||
;;
|
||||
--sync-id=*)
|
||||
SYNC_ID=${i##*=}
|
||||
opts=$opts" --sync-id=\"$SYNC_ID\""
|
||||
;;
|
||||
--on-changes)
|
||||
sync_on_changes=1
|
||||
;;
|
||||
|
|
@ -1708,9 +1737,15 @@ opts="${opts# *}"
|
|||
CheckEnvironment
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
|
||||
## Here we set default options for quicksync tasks when no configuration file is provided.
|
||||
|
||||
if [ $quick_sync -eq 2 ]
|
||||
then
|
||||
SYNC_ID="quicksync task"
|
||||
if [ "$SYNC_ID" == "" ]
|
||||
then
|
||||
SYNC_ID="quicksync task"
|
||||
fi
|
||||
MINIMUM_SPACE=1024
|
||||
REMOTE_SYNC=no
|
||||
CONFLICT_BACKUP_DAYS=30
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
###### Osync - Rsync based two way sync engine with fault tolerance
|
||||
###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
||||
###### Config file rev 2303201401
|
||||
###### Config file rev 0805201402
|
||||
|
||||
## ---------- GENERAL OPTIONS
|
||||
|
||||
|
|
@ -12,7 +12,7 @@ SYNC_ID="sync_test"
|
|||
## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one.
|
||||
MASTER_SYNC_DIR="/home/git/osync/test/dir1"
|
||||
#SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
|
||||
SLAVE_SYNC_DIR=""ssh://user@host.com:22//path/to/dir2"
|
||||
SLAVE_SYNC_DIR="ssh://user@host.com:22//path/to/dir2"
|
||||
## If slave replica is a remote directory, you must specifiy a RSA key (please use full path). Please see documentation for further information.
|
||||
SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa"
|
||||
|
||||
|
|
@ -25,6 +25,9 @@ LOGFILE=""
|
|||
## List of directories to exclude from sync on both sides (rsync patterns, wildcards work).
|
||||
## Paths are relative to sync dirs. List elements are separated by a semicolon.
|
||||
RSYNC_EXCLUDE_PATTERN="tmp;archives"
|
||||
## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
|
||||
## Paths are relative to sync dirs. One element per line.
|
||||
RSYNC_EXCLUDE_FROM="exclude.list"
|
||||
## List elements separator char. You may set an alternative seperator char for your directories lists above.
|
||||
PATH_SEPARATOR_CHAR=";"
|
||||
|
||||
|
|
@ -61,6 +64,8 @@ RSYNC_REMOTE_PATH=""
|
|||
PRESERVE_ACL=no
|
||||
## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors.
|
||||
PRESERVE_XATTR=no
|
||||
## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
|
||||
PRESERVE_HARDLINKS=yes
|
||||
|
||||
## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression.
|
||||
RSYNC_COMPRESS=yes
|
||||
|
|
|
|||
Loading…
Reference in New Issue