WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into no-resume

This commit is contained in:
Orsiris de Jong 2017-05-30 14:50:21 +02:00 committed by GitHub
parent 45cb042fa0 5c60e7a9a2
commit 6a4b3701d0
2 changed files with 287 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

362
dev/n_osync.sh
View File

@ -3,9 +3,10 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong" AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.1-rc1 PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017053002 PROGRAM_BUILD=2017053005
IS_STABLE=yes IS_STABLE=no
##### Execution order #__WITH_PARANOIA_DEBUG ##### Execution order #__WITH_PARANOIA_DEBUG
##### Function Name Is parallel #__WITH_PARANOIA_DEBUG ##### Function Name Is parallel #__WITH_PARANOIA_DEBUG
@ -24,10 +25,11 @@ IS_STABLE=yes
# Sync no #__WITH_PARANOIA_DEBUG # Sync no #__WITH_PARANOIA_DEBUG
# treeList yes #__WITH_PARANOIA_DEBUG # treeList yes #__WITH_PARANOIA_DEBUG
# treeList yes #__WITH_PARANOIA_DEBUG # treeList yes #__WITH_PARANOIA_DEBUG
# _getFileCtimeMtime yes #__WITH_PARANOIA_DEBUG
# _getFileCtimeMtime yes #__WITH_PARANOIA_DEBUG
# deleteList yes #__WITH_PARANOIA_DEBUG # deleteList yes #__WITH_PARANOIA_DEBUG
# deleteList yes #__WITH_PARANOIA_DEBUG # deleteList yes #__WITH_PARANOIA_DEBUG
# syncAttrs no #__WITH_PARANOIA_DEBUG # syncAttrs no #__WITH_PARANOIA_DEBUG
# _getFileCtimeMtime yes #__WITH_PARANOIA_DEBUG
# syncUpdate no #__WITH_PARANOIA_DEBUG # syncUpdate no #__WITH_PARANOIA_DEBUG
# syncUpdate no #__WITH_PARANOIA_DEBUG # syncUpdate no #__WITH_PARANOIA_DEBUG
# deletionPropagation yes #__WITH_PARANOIA_DEBUG # deletionPropagation yes #__WITH_PARANOIA_DEBUG
@ -850,6 +852,72 @@ ENDSSH
fi fi
} }
#WIP function that takes treeList files and gets ctime and mtime for each file, then compares those files to create the conflict file list
function timestampList {
local replicaPath="${1}" # path to the replica for which a tree needs to be constructed
local replicaType="${2}" # replica type: initiator, target
local fileList="${3}" # List of files to get timestamps for
local timestampFilename="${4}" # filename to output timestamp list (will be prefixed with $replicaType)
__CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG
local retval
local escapedReplicaPath
local rsyncCmd
Logger "Getting file stats for $replicaType replica." "NOTICE"
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
timestampFilename="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
timestampFilename="${INITIATOR[$__type]}"
else
Logger "Bogus replicaType in [${FUNCNAME[0]}]." "CRITICAL"
exit 1
fi
Logger "Creating $replicaType replica file list [$replicaPath]." "NOTICE"
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
_getFileCtimeMtimeRemote "$replicaPath" "$replicaType" "$fileList" "$timestampFilename"
else
_getFileCtimeMtimeLocal "$replicaPath" "$replicaType" "$fileList" "$timestampFilename"
fi
}
#WIP
function conflictList {
return 0
local replicaPath="${1}" # path to the replica for which a tree needs to be constructed
local replicaType="${2}" # replica type: initiator, target
local timestampCurrentFilename="${3}" # filename of current timestamp list (will be prefixed with $replicaType)
local timestampPreviousFilename="${3}" # filename of previous timestamp list (will be prefixed with $replicaType)
local conflictFilename="{4}" # filename of conflicts
__CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG
local retval
local escapedReplicaPath
local rsyncCmd
Logger "Creating conflict list for $replicaType replica." "NOTICE"
# Compare if previous exists
}
#WIP
function renameConflictList {
return 0
local timestampCurrentFilename="${3}" # filename of current timestamp list (will be prefixed with $replicaType)
local timestampPreviousFilename="${3}" # filename of previous timestamp list (will be prefixed with $replicaType)
__CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG
# Rename files
}
# rsync does sync with mtime, but file attribute modifications only change ctime. # rsync does sync with mtime, but file attribute modifications only change ctime.
# Hence, detect newer ctime on the replica that gets updated first with CONFLICT_PREVALANCE and update all newer file attributes on this replica before real update # Hence, detect newer ctime on the replica that gets updated first with CONFLICT_PREVALANCE and update all newer file attributes on this replica before real update
function syncAttrs { function syncAttrs {
@ -1329,14 +1397,17 @@ function deletionPropagation {
fi fi
} }
###### Sync function in 6 steps ###### Sync function in 9 steps
###### ######
###### Step 0a & 0b: Create current file list of replicas ###### Step 0a & 0b: Create current file list of replicas
###### Step 1a & 1b: Create deleted file list of replicas ###### Step 1a & 1b: Create deleted file list of replicas
###### Step 3: Update file attributes ###### Step 2a & 2b: Create ctime & mtime file list of replicas
###### Step 3a & 3b: Update replicas ###### Step 3a & 3b: Merge conflict file list
###### Step 4a & 4b: Propagate deletions on replicas ###### Step 4: Update file attributes
###### Step 5a & 5b: Create after run file list of replicas ###### Step 5a & 5b: Update replicas
###### Step 6a & 6b: Propagate deletions on replicas
###### Step 7a & 8b: Create after run file list of replicas
###### Step 8: Rename timestamp files from current to previous
function Sync { function Sync {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -1490,22 +1561,43 @@ function Sync {
fi fi
fi fi
## Step 2 ## Step 2a & 2b
if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" & timestampList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__type]}${INITIATOR[$__timestampsCurrentFile]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false initiatorPid="$!"
if [ $? -ne 0 ]; then fi
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}"
exit 1
else
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[3]}"
resumeTarget="${SYNC_ACTION[3]}"
if [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
timestampList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${TARGET[$__type]}${TARGET[$__timestampsCurrentFile]}" &
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true
elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true
fi
done
if [ $initiatorFail == false ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
fi fi
if [ $targetFail == false ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
fi
exit 1
else else
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -1516,67 +1608,136 @@ function Sync {
## Step 3a & 3b ## Step 3a & 3b
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then
conflictList "${INITIATOR[$__type]}" &
initiatorPid="$!"
fi
if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
conflictList "${TARGET[$__type]}" &
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true
elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true
fi
done
if [ $initiatorFail == false ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
fi
if [ $targetFail == false ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
fi
exit 1
else
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[4]}"
resumeTarget="${SYNC_ACTION[4]}"
fi
fi
## Step 4
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
exit 1
else
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
resumeTarget="${SYNC_ACTION[5]}"
fi
else
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
resumeTarget="${SYNC_ACTION[5]}"
fi
fi
## Step 5a & 5b
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then
if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[3]}" resumeTarget="${SYNC_ACTION[5]}"
exit 1 exit 1
else else
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[4]}" resumeTarget="${SYNC_ACTION[6]}"
fi fi
fi fi
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[3]}" resumeInitiator="${SYNC_ACTION[5]}"
exit 1 exit 1
else else
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[4]}" resumeInitiator="${SYNC_ACTION[6]}"
fi fi
fi fi
else else
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[3]}" resumeInitiator="${SYNC_ACTION[5]}"
exit 1 exit 1
else else
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[4]}" resumeInitiator="${SYNC_ACTION[6]}"
fi fi
fi fi
if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[3]}" resumeTarget="${SYNC_ACTION[5]}"
exit 1 exit 1
else else
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[4]}" resumeTarget="${SYNC_ACTION[6]}"
fi fi
fi fi
fi fi
fi fi
## Step 4a & 4b ## Step 6a & 6b
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[6]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[6]}" ]; then
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[6]}" ]; then
deletionPropagation "${INITIATOR[$__type]}" & deletionPropagation "${INITIATOR[$__type]}" &
initiatorPid="$!" initiatorPid="$!"
fi fi
if [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[6]}" ]; then
deletionPropagation "${TARGET[$__type]}" & deletionPropagation "${TARGET[$__type]}" &
targetPid="$!" targetPid="$!"
fi fi
@ -1589,40 +1750,40 @@ function Sync {
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
pid=${pid%:*} pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true initiatorFail=true
elif [ "$pid" == "$targetPid" ]; then elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true targetFail=true
fi fi
done done
if [ $initiatorFail == false ]; then if [ $initiatorFail == false ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__initiatorLastActionFile]}"
fi fi
if [ $targetFail == false ]; then if [ $targetFail == false ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__targetLastActionFile]}"
fi fi
exit 1 exit 1
else else
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}" resumeInitiator="${SYNC_ACTION[7]}"
resumeTarget="${SYNC_ACTION[5]}" resumeTarget="${SYNC_ACTION[7]}"
fi fi
fi fi
## Step 5a & 5b ## Step 7a & 7b
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[7]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[7]}" ]; then
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[7]}" ]; then
treeList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__treeAfterFile]}" & treeList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__treeAfterFile]}" &
initiatorPid="$!" initiatorPid="$!"
fi fi
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[7]}" ]; then
treeList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__treeAfterFile]}" & treeList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__treeAfterFile]}" &
targetPid="$!" targetPid="$!"
fi fi
@ -1635,28 +1796,52 @@ function Sync {
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
pid=${pid%:*} pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true initiatorFail=true
elif [ "$pid" == "$targetPid" ]; then elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true targetFail=true
fi fi
done done
if [ $initiatorFail == false ]; then if [ $initiatorFail == false ]; then
echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__initiatorLastActionFile]}"
fi fi
if [ $targetFail == false ]; then if [ $targetFail == false ]; then
echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__targetLastActionFile]}"
fi fi
exit 1 exit 1
else else
echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[6]}" resumeInitiator="${SYNC_ACTION[8]}"
resumeTarget="${SYNC_ACTION[6]}" resumeTarget="${SYNC_ACTION[8]}"
fi
fi
# Step 8 #WIP adapt to last function
if [ "$resumeInitiator" == "${SYNC_ACTION[8]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[8]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
renameTimestampFiles "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__targetLastActionFile]}"
exit 1
else
echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
resumeTarget="${SYNC_ACTION[5]}"
fi
else
echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[9]}"
resumeTarget="${SYNC_ACTION[9]}"
fi fi
fi fi
@ -1954,6 +2139,9 @@ function Init {
readonly __deletedListFile=13 readonly __deletedListFile=13
readonly __failedDeletedListFile=14 readonly __failedDeletedListFile=14
readonly __successDeletedListFile=15 readonly __successDeletedListFile=15
readonly __timestampCurrentFile=15
readonly __timestampPreviousFile=16
readonly __conflictListFile=17
INITIATOR=() INITIATOR=()
INITIATOR[$__type]='initiator' INITIATOR[$__type]='initiator'
@ -1972,6 +2160,9 @@ function Init {
INITIATOR[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" INITIATOR[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix"
INITIATOR[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" INITIATOR[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
INITIATOR[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" INITIATOR[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampPreviousFile]="-timestamps-previous-$INSTANCE_ID$drySuffix"
INITIATOR[$__conflictListfile]="conflicts-$INSTANCE_ID$drySuffix"
TARGET=() TARGET=()
TARGET[$__type]='target' TARGET[$__type]='target'
@ -1990,6 +2181,9 @@ function Init {
TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused
TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
TARGET[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix"
TARGET[$__timestampPreviousFile]="-timestamps-previous-$INSTANCE_ID$drySuffix"
TARGET[$__conflictListfile]="conflicts-$INSTANCE_ID$drySuffix"
PARTIAL_DIR="${INITIATOR[$__partialDir]}" PARTIAL_DIR="${INITIATOR[$__partialDir]}"
@ -2020,13 +2214,19 @@ function Init {
TARGET_BACKUP="" TARGET_BACKUP=""
fi fi
#WIP: change resume numbers when new conflict function will be done
#WIP: conflict list is not mandatory, but is still needed for acl resolution
#WIP: syncAttrs must move the file list to sub function, which checks which kind of file list to use
SYNC_ACTION=( SYNC_ACTION=(
'replica-tree' 'replica-tree'
'deleted-list' 'deleted-list'
'ctime-mtime-list'
'conflict-list'
'sync_attrs' 'sync_attrs'
'update-replica' 'update-replica'
'delete-propagation' 'delete-propagation'
'replica-tree-after' 'replica-tree-after'
'rename-timestamp-files'
'synced' 'synced'
) )
} }
@ -2055,18 +2255,20 @@ function Usage {
echo "or osync.sh --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]" echo "or osync.sh --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "" echo ""
echo "[OPTIONS]" echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing" echo "--dry Will run osync without actually doing anything; just testing"
echo "--no-prefix Will suppress time / date suffix from output" echo "--no-prefix Will suppress time / date suffix from output"
echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--silent Will run osync without any output to stdout, used for cron jobs"
echo "--errors-only Output only errors (can be combined with silent or verbose)" echo "--errors-only Output only errors (can be combined with silent or verbose)"
echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--summary Outputs a list of transferred / deleted files at the end of the run"
echo "--verbose Increases output" echo "--log-conflicts Outputs a list of conflicted files"
echo "--stats Adds rsync transfer statistics to verbose output" echo "--verbose Increases output"
echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" echo "--stats Adds rsync transfer statistics to verbose output"
echo "--no-maxtime Disables any soft and hard execution time checks" echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)"
echo "--force-unlock Will override any existing active or dead locks on initiator and target replica" echo "--no-maxtime Disables any soft and hard execution time checks"
echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead" echo "--force-unlock Will override any existing active or dead locks on initiator and target replica"
echo "--no-resume Do not try to resume a failed run. By default, execution is resumed once" echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead"
echo "--no-resume Do not try to resume a failed run. By default, execution is resumed once"
echo "" echo ""
echo "[QUICKSYNC OPTIONS]" echo "[QUICKSYNC OPTIONS]"
echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)" echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)"
@ -2259,6 +2461,10 @@ for i in "$@"; do
opts=$opts" --summary" opts=$opts" --summary"
_SUMMARY=true _SUMMARY=true
;; ;;
--log-conflicts)
_LOG_CONFLICTS=true
opts=$opts" --log-conflicts"
;;
--no-prefix) --no-prefix)
opts=$opts" --no-prefix" opts=$opts" --no-prefix"
_LOGGER_PREFIX="" _LOGGER_PREFIX=""

4
sync.conf.example
View File

@ -2,7 +2,7 @@
###### osync - Rsync based two way sync engine with fault tolerance ###### osync - Rsync based two way sync engine with fault tolerance
###### (C) 2013-2017 by Orsiris de Jong (www.netpower.fr) ###### (C) 2013-2017 by Orsiris de Jong (www.netpower.fr)
###### osync v1.1x / v1.2x config file rev 2017020801 ###### osync v1.1x / v1.2x config file rev 2017053001
## ---------- GENERAL OPTIONS ## ---------- GENERAL OPTIONS
@ -127,6 +127,8 @@ MAX_WAIT=7200
## ---------- BACKUP AND DELETION OPTIONS ## ---------- BACKUP AND DELETION OPTIONS
## Log a list of conflictual files
LOG_CONFLICTS=yes
## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups ## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups
CONFLICT_BACKUP=yes CONFLICT_BACKUP=yes
## Keep multiple backup versions of the same file. Warning, This can be very space consuming. ## Keep multiple backup versions of the same file. Warning, This can be very space consuming.