WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented --alert-conflicts options

This commit is contained in:
deajan 2017-06-05 18:53:15 +02:00
parent be3b3d39b6
commit 7a24cf49cd
2 changed files with 36 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
dev/n_osync.sh
View File

@ -3,12 +3,17 @@
#TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ? #TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ?
#TODO check if _getCtimeMtime | sort removal needs to be backported #TODO check if _getCtimeMtime | sort removal needs to be backported
#TODO backport treeList sed -r sed -E 's/^.{10} +[0-9]+ [0-9/]{10} [0-9:]{8} //' fix && _getFileCtimeMtime* IFS read fix #TODO backport treeList sed -r sed -E 's/^.{10} +[0-9]+ [0-9/]{10} [0-9:]{8} //' fix && _getFileCtimeMtime* IFS read fix
#TODO LANG=C... backport to v1.2.1 and v1.1
#TODO: conflict list is not mandatory, but is still needed for acl resolution
#TODO: syncAttrs must move the file list to sub function, which checks which kind of file list to use
#TODO: nosuffix in order not to overwrite on dryrun
#TODO: double .xz extension when sending email alert with attachment
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong" AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.2-dev PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017060501 PROGRAM_BUILD=2017060503
IS_STABLE=no IS_STABLE=no
@ -822,7 +827,6 @@ function _getFileCtimeMtimeRemote {
local retval local retval
local cmd local cmd
#WIP check if the following works with env remote token on top of cat
cmd='cat "'$fileList'" | '$SSH_CMD' "env _REMOTE_TOKEN=$_REMOTE_TOKEN cat > \".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP\""' cmd='cat "'$fileList'" | '$SSH_CMD' "env _REMOTE_TOKEN=$_REMOTE_TOKEN cat > \".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP\""'
Logger "Launching command [$cmd]." "DEBUG" Logger "Launching command [$cmd]." "DEBUG"
eval "$cmd" eval "$cmd"
@ -836,7 +840,6 @@ function _getFileCtimeMtimeRemote {
return $retval return $retval
fi fi
#WIP LANG=C... backport to v1.2.1 and v1.1
$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \ $SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
@ -897,7 +900,6 @@ function timestampList {
return $retval return $retval
} }
#WIP
function conflictList { function conflictList {
local timestampCurrentFilename="${1}" # filename of current timestamp list (will be prefixed with $replicaType) local timestampCurrentFilename="${1}" # filename of current timestamp list (will be prefixed with $replicaType)
local timestampAfterFilename="${2}" # filename of previous timestamp list (will be prefixed with $replicaType) local timestampAfterFilename="${2}" # filename of previous timestamp list (will be prefixed with $replicaType)
@ -1023,7 +1025,6 @@ function syncAttrs {
fi fi
fi fi
#WIP: replace 4th argument with some state file
Logger "Getting ctimes for pending files on initiator." "NOTICE" Logger "Getting ctimes for pending files on initiator." "NOTICE"
_getFileCtimeMtimeLocal "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" & _getFileCtimeMtimeLocal "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" &
pids="$!" pids="$!"
@ -1670,7 +1671,7 @@ function Sync {
## Step 3a & 3b ## Step 3a & 3b
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]] || [ "$LOG_CONFLICTS" == "yes" ]; then if [ "$LOG_CONFLICTS" == "yes" ]; then
conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFile]}" "${INITIATOR[$__conflictListFile]}" & conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFile]}" "${INITIATOR[$__conflictListFile]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
@ -2107,13 +2108,26 @@ function Summary {
function LogConflicts { function LogConflicts {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
local subject
local body
( (
_LOGGER_PREFIX="" _LOGGER_PREFIX=""
Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS"
if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then
echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
while read -r line; do while read -r line; do
Logger "${INITIATOR[$__replicaDir]}$(echo $line | awk -F';' '{print $1}') -- ${TARGET[$__replicaDir]}$(echo $line | awk -F';' '{print $1}')" "ALWAYS" echo "${INITIATOR[$__replicaDir]}$(echo $line | awk -F';' '{print $1}') -- ${TARGET[$__replicaDir]}$(echo $line | awk -F';' '{print $1}')" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
done < "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" done < "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP"
Logger "$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})" "ALWAYS"
if [ "$ALERT_CONFLICTS" == "yes" ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
fi fi
) )
} }
@ -2225,7 +2239,8 @@ function Init {
readonly __successDeletedListFile=15 readonly __successDeletedListFile=15
readonly __timestampCurrentFile=16 readonly __timestampCurrentFile=16
readonly __timestampAfterFile=17 readonly __timestampAfterFile=17
readonly __conflictListFile=18 readonly __timestampAfterFileNoSuffix=18
readonly __conflictListFile=19
INITIATOR=() INITIATOR=()
INITIATOR[$__type]='initiator' INITIATOR[$__type]='initiator'
@ -2246,6 +2261,7 @@ function Init {
INITIATOR[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" INITIATOR[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix" INITIATOR[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix" INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix" INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
TARGET=() TARGET=()
@ -2261,12 +2277,14 @@ function Init {
TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused
#WIP NoSuffix file to add to timestamp
TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused
TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused
TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
TARGET[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix" TARGET[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix"
TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix" TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
TARGET[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix" TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
PARTIAL_DIR="${INITIATOR[$__partialDir]}" PARTIAL_DIR="${INITIATOR[$__partialDir]}"
@ -2298,9 +2316,6 @@ function Init {
TARGET_BACKUP="" TARGET_BACKUP=""
fi fi
#WIP: change resume numbers when new conflict function will be done
#WIP: conflict list is not mandatory, but is still needed for acl resolution
#WIP: syncAttrs must move the file list to sub function, which checks which kind of file list to use
SYNC_ACTION=( SYNC_ACTION=(
'replica-tree' 'replica-tree'
'deleted-list' 'deleted-list'
@ -2345,6 +2360,7 @@ function Usage {
echo "--errors-only Output only errors (can be combined with silent or verbose)" echo "--errors-only Output only errors (can be combined with silent or verbose)"
echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--summary Outputs a list of transferred / deleted files at the end of the run"
echo "--log-conflicts Outputs a list of conflicted files" echo "--log-conflicts Outputs a list of conflicted files"
echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)"
echo "--verbose Increases output" echo "--verbose Increases output"
echo "--stats Adds rsync transfer statistics to verbose output" echo "--stats Adds rsync transfer statistics to verbose output"
echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)"
@ -2448,6 +2464,7 @@ INITIATOR_LOCK_FILE_EXISTS=false
TARGET_LOCK_FILE_EXISTS=false TARGET_LOCK_FILE_EXISTS=false
FORCE_UNLOCK=false FORCE_UNLOCK=false
LOG_CONFLICTS="no" LOG_CONFLICTS="no"
ALERT_CONFLICTS="no"
no_maxtime=false no_maxtime=false
opts="" opts=""
ERROR_ALERT=false ERROR_ALERT=false
@ -2550,6 +2567,11 @@ for i in "$@"; do
LOG_CONFLICTS="yes" LOG_CONFLICTS="yes"
opts=$opts" --log-conflicts" opts=$opts" --log-conflicts"
;; ;;
--alert-conflicts)
ALERT_CONFLICTS="yes"
LOG_CONFLICTS="yes"
opts=$opts" --alert-conflicts"
;;
--no-prefix) --no-prefix)
opts=$opts" --no-prefix" opts=$opts" --no-prefix"
_LOGGER_PREFIX="" _LOGGER_PREFIX=""

4
sync.conf.example
View File

@ -2,7 +2,7 @@
###### osync - Rsync based two way sync engine with fault tolerance ###### osync - Rsync based two way sync engine with fault tolerance
###### (C) 2013-2017 by Orsiris de Jong (www.netpower.fr) ###### (C) 2013-2017 by Orsiris de Jong (www.netpower.fr)
###### osync v1.1x / v1.2x config file rev 2017053001 ###### osync v1.1x / v1.2x config file rev 2017060501
## ---------- GENERAL OPTIONS ## ---------- GENERAL OPTIONS
@ -129,6 +129,8 @@ MAX_WAIT=7200
## Log a list of conflictual files ## Log a list of conflictual files
LOG_CONFLICTS=yes LOG_CONFLICTS=yes
## Send an email when conflictual files are found (implies LOG_CONFLICTS)
ALERT_CONFLICTS=no
## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups ## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups
CONFLICT_BACKUP=yes CONFLICT_BACKUP=yes
## Keep multiple backup versions of the same file. Warning, This can be very space consuming. ## Keep multiple backup versions of the same file. Warning, This can be very space consuming.