From a32b5f832176d2f605bc4d95de745ad795dc3571 Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 09:15:48 +0200 Subject: [PATCH 1/7] Bumped version to next dev --- dev/n_osync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 70c5746..90bb391 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -3,9 +3,9 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2.1-rc1 +PROGRAM_VERSION=1.2.2-dev PROGRAM_BUILD=2017053010 -IS_STABLE=yes +IS_STABLE=no ##### Execution order #__WITH_PARANOIA_DEBUG ##### Function Name Is parallel #__WITH_PARANOIA_DEBUG From 0b442e2bcf78417fffc9664580d671b8e8f233d1 Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 09:19:47 +0200 Subject: [PATCH 2/7] Updated work comments --- dev/n_osync.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 90bb391..792a4bc 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -24,10 +24,11 @@ IS_STABLE=no # Sync no #__WITH_PARANOIA_DEBUG # treeList yes #__WITH_PARANOIA_DEBUG # treeList yes #__WITH_PARANOIA_DEBUG +# _getFileCtimeMtime yes #__WITH_PARANOIA_DEBUG +# _getFileCtimeMtime yes #__WITH_PARANOIA_DEBUG # deleteList yes #__WITH_PARANOIA_DEBUG # deleteList yes #__WITH_PARANOIA_DEBUG # syncAttrs no #__WITH_PARANOIA_DEBUG -# _getFileCtimeMtime yes #__WITH_PARANOIA_DEBUG # syncUpdate no #__WITH_PARANOIA_DEBUG # syncUpdate no #__WITH_PARANOIA_DEBUG # deletionPropagation yes #__WITH_PARANOIA_DEBUG From 908c9af888daca84dce73a3566459c621fcb9f6c Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 09:29:16 +0200 Subject: [PATCH 3/7] Added --log-conflicts option --- dev/n_osync.sh | 31 ++++++++++++++++++------------- sync.conf.example | 4 +++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 792a4bc..f39186c 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2.2-dev -PROGRAM_BUILD=2017053010 +PROGRAM_BUILD=2017053001 IS_STABLE=no ##### Execution order #__WITH_PARANOIA_DEBUG @@ -2056,17 +2056,18 @@ function Usage { echo "or osync.sh --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]" echo "" echo "[OPTIONS]" - echo "--dry Will run osync without actually doing anything; just testing" - echo "--no-prefix Will suppress time / date suffix from output" - echo "--silent Will run osync without any output to stdout, used for cron jobs" - echo "--errors-only Output only errors (can be combined with silent or verbose)" - echo "--summary Outputs a list of transferred / deleted files at the end of the run" - echo "--verbose Increases output" - echo "--stats Adds rsync transfer statistics to verbose output" - echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" - echo "--no-maxtime Disables any soft and hard execution time checks" - echo "--force-unlock Will override any existing active or dead locks on initiator and target replica" - echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead" + echo "--dry Will run osync without actually doing anything; just testing" + echo "--no-prefix Will suppress time / date suffix from output" + echo "--silent Will run osync without any output to stdout, used for cron jobs" + echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--summary Outputs a list of transferred / deleted files at the end of the run" + echo "--log-conflicts Outputs a list of conflicted files" + echo "--verbose Increases output" + echo "--stats Adds rsync transfer statistics to verbose output" + echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" + echo "--no-maxtime Disables any soft and hard execution time checks" + echo "--force-unlock Will override any existing active or dead locks on initiator and target replica" + echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead" echo "" echo "[QUICKSYNC OPTIONS]" echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)" @@ -2248,11 +2249,15 @@ for i in "$@"; do _NOLOCKS=true ;; --errors-only) - _LOGGER_ERR_ONLY=true + LOGGER_ERR_ONLY="yes" ;; --summary) _SUMMARY=true ;; + --log-conflicts) + _LOG_CONFLICTS=true + opts=$opts" --log-conflicts" + ;; --no-prefix) _LOGGER_PREFIX="" ;; diff --git a/sync.conf.example b/sync.conf.example index 3645a36..f94be19 100644 --- a/sync.conf.example +++ b/sync.conf.example @@ -2,7 +2,7 @@ ###### osync - Rsync based two way sync engine with fault tolerance ###### (C) 2013-2017 by Orsiris de Jong (www.netpower.fr) -###### osync v1.1x / v1.2x config file rev 2017020801 +###### osync v1.1x / v1.2x config file rev 2017053001 ## ---------- GENERAL OPTIONS @@ -127,6 +127,8 @@ MAX_WAIT=7200 ## ---------- BACKUP AND DELETION OPTIONS +## Log a list of conflictual files +LOG_CONFLICTS=yes ## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups CONFLICT_BACKUP=yes ## Keep multiple backup versions of the same file. Warning, This can be very space consuming. From b541a7c98ea21ad815769ee2a103cadca7eb30de Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 11:14:11 +0200 Subject: [PATCH 4/7] WIP: detect conflict files --- dev/n_osync.sh | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 5c122fb..1599717 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2.2-dev -PROGRAM_BUILD=2017053002 +PROGRAM_BUILD=2017053003 IS_STABLE=no @@ -852,6 +852,22 @@ ENDSSH fi } +#WIP function that takes treeList files and gets ctime and mtime for each file, then compares those files to create the conflict file list +function conflictFileList { + local replicaPath="${1}" # path to the replica for which a tree needs to be constructed + local replicaType="${2}" # replica type: initiator, target + local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) + local conflictFileName="{4}" + + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG + + local retval + local escapedReplicaPath + local rsyncCmd + #_getCtimeMtime + #compare +} + # rsync does sync with mtime, but file attribute modifications only change ctime. # Hence, detect newer ctime on the replica that gets updated first with CONFLICT_PREVALANCE and update all newer file attributes on this replica before real update function syncAttrs { @@ -1335,10 +1351,11 @@ function deletionPropagation { ###### ###### Step 0a & 0b: Create current file list of replicas ###### Step 1a & 1b: Create deleted file list of replicas +###### Step 2a & 2b: Create conflict file list of replicas ###### Step 3: Update file attributes -###### Step 3a & 3b: Update replicas -###### Step 4a & 4b: Propagate deletions on replicas -###### Step 5a & 5b: Create after run file list of replicas +###### Step 4a & 4b: Update replicas +###### Step 5a & 5b: Propagate deletions on replicas +###### Step 6a & 6b: Create after run file list of replicas function Sync { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -2022,9 +2039,13 @@ function Init { TARGET_BACKUP="" fi + #WIP: change resume numbers when new conflict function will be done + #WIP: conflict list is not mandatory, but is still needed for acl resolution + #WIP: syncAttrs must move the file list to sub function, which checks which kind of file list to use SYNC_ACTION=( 'replica-tree' 'deleted-list' + 'conflict-list' 'sync_attrs' 'update-replica' 'delete-propagation' From f98a9a0748aedc389628e8b7cabfc9524d46df51 Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 12:16:59 +0200 Subject: [PATCH 5/7] WIP: getting conflict list of files --- dev/n_osync.sh | 242 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 178 insertions(+), 64 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 1599717..cba8861 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2.2-dev -PROGRAM_BUILD=2017053003 +PROGRAM_BUILD=2017053004 IS_STABLE=no @@ -853,7 +853,7 @@ ENDSSH } #WIP function that takes treeList files and gets ctime and mtime for each file, then compares those files to create the conflict file list -function conflictFileList { +function timestampList { local replicaPath="${1}" # path to the replica for which a tree needs to be constructed local replicaType="${2}" # replica type: initiator, target local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) @@ -864,6 +864,28 @@ function conflictFileList { local retval local escapedReplicaPath local rsyncCmd + + Logger "Getting file stats for $replicaType replica." "NOTICE" + + #_getCtimeMtime + #compare +} + +#WIP +function conflictList { + local replicaPath="${1}" # path to the replica for which a tree needs to be constructed + local replicaType="${2}" # replica type: initiator, target + local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) + local conflictFileName="{4}" + + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG + + local retval + local escapedReplicaPath + local rsyncCmd + + Logger "Creating conflict list for $replicaType replica." "NOTICE" + #_getCtimeMtime #compare } @@ -1351,11 +1373,12 @@ function deletionPropagation { ###### ###### Step 0a & 0b: Create current file list of replicas ###### Step 1a & 1b: Create deleted file list of replicas -###### Step 2a & 2b: Create conflict file list of replicas -###### Step 3: Update file attributes -###### Step 4a & 4b: Update replicas -###### Step 5a & 5b: Propagate deletions on replicas -###### Step 6a & 6b: Create after run file list of replicas +###### Step 2a & 2b: Create ctime & mtime file list of replicas +###### Step 3a & 3b: Merge conflict file list +###### Step 4: Update file attributes +###### Step 5a & 5b: Update replicas +###### Step 6a & 6b: Propagate deletions on replicas +###### Step 7a & 8b: Create after run file list of replicas function Sync { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1509,22 +1532,43 @@ function Sync { fi fi - ## Step 2 + ## Step 2a & 2b if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then - if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then - syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false - if [ $? -ne 0 ]; then - echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}" - echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}" - exit 1 - else - echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" - echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeInitiator="${SYNC_ACTION[3]}" - resumeTarget="${SYNC_ACTION[3]}" + if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ]; then + timestampList "${INITIATOR[$__type]}" & + initiatorPid="$!" + fi + if [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then + timestampList "${TARGET[$__type]}" & + targetPid="$!" + fi + + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false + if [ $? -ne 0 ]; then + IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" + initiatorFail=false + targetFail=false + for pid in "${pidArray[@]}"; do + pid=${pid%:*} + if [ "$pid" == "$initiatorPid" ]; then + echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}" + initiatorFail=true + elif [ "$pid" == "$targetPid" ]; then + echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}" + targetFail=true + fi + done + + if [ $initiatorFail == false ]; then + echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" fi + + if [ $targetFail == false ]; then + echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" + fi + + exit 1 else echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" @@ -1535,67 +1579,136 @@ function Sync { ## Step 3a & 3b if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then + if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then + conflictList "${INITIATOR[$__type]}" & + initiatorPid="$!" + fi + + if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then + conflictList "${TARGET[$__type]}" & + targetPid="$!" + fi + + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false + if [ $? -ne 0 ]; then + IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" + initiatorFail=false + targetFail=false + for pid in "${pidArray[@]}"; do + pid=${pid%:*} + if [ "$pid" == "$initiatorPid" ]; then + echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" + initiatorFail=true + elif [ "$pid" == "$targetPid" ]; then + echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" + targetFail=true + fi + done + + if [ $initiatorFail == false ]; then + echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" + fi + + if [ $targetFail == false ]; then + echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" + fi + + exit 1 + else + echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[4]}" + resumeTarget="${SYNC_ACTION[4]}" + fi + fi + + ## Step 4 + if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then + if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then + syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false + if [ $? -ne 0 ]; then + echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" + exit 1 + else + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[5]}" + resumeTarget="${SYNC_ACTION[5]}" + + fi + else + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[5]}" + resumeTarget="${SYNC_ACTION[5]}" + fi + fi + + ## Step 5a & 5b + if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then - if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then + if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? -ne 0 ]; then - echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeTarget="${SYNC_ACTION[3]}" + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeTarget="${SYNC_ACTION[5]}" exit 1 else - echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeTarget="${SYNC_ACTION[4]}" + echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeTarget="${SYNC_ACTION[6]}" fi fi - if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then + if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? -ne 0 ]; then - echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" - resumeInitiator="${SYNC_ACTION[3]}" + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" + resumeInitiator="${SYNC_ACTION[5]}" exit 1 else - echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" - resumeInitiator="${SYNC_ACTION[4]}" + echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" + resumeInitiator="${SYNC_ACTION[6]}" fi fi else - if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ]; then + if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? -ne 0 ]; then - echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" - resumeInitiator="${SYNC_ACTION[3]}" + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" + resumeInitiator="${SYNC_ACTION[5]}" exit 1 else - echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" - resumeInitiator="${SYNC_ACTION[4]}" + echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" + resumeInitiator="${SYNC_ACTION[6]}" fi fi - if [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then + if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false if [ $? -ne 0 ]; then - echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeTarget="${SYNC_ACTION[3]}" + echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeTarget="${SYNC_ACTION[5]}" exit 1 else - echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeTarget="${SYNC_ACTION[4]}" + echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeTarget="${SYNC_ACTION[6]}" fi fi fi fi - ## Step 4a & 4b - if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then - if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ]; then + ## Step 6a & 6b + if [ "$resumeInitiator" == "${SYNC_ACTION[6]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[6]}" ]; then + if [ "$resumeInitiator" == "${SYNC_ACTION[6]}" ]; then deletionPropagation "${INITIATOR[$__type]}" & initiatorPid="$!" fi - if [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then + if [ "$resumeTarget" == "${SYNC_ACTION[6]}" ]; then deletionPropagation "${TARGET[$__type]}" & targetPid="$!" fi @@ -1608,40 +1721,40 @@ function Sync { for pid in "${pidArray[@]}"; do pid=${pid%:*} if [ "$pid" == "$initiatorPid" ]; then - echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" initiatorFail=true elif [ "$pid" == "$targetPid" ]; then - echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" + echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" targetFail=true fi done if [ $initiatorFail == false ]; then - echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__initiatorLastActionFile]}" fi if [ $targetFail == false ]; then - echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" + echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__targetLastActionFile]}" fi exit 1 else - echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" - echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeInitiator="${SYNC_ACTION[5]}" - resumeTarget="${SYNC_ACTION[5]}" + echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[7]}" + resumeTarget="${SYNC_ACTION[7]}" fi fi - ## Step 5a & 5b - if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then - if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then + ## Step 7a & 7b + if [ "$resumeInitiator" == "${SYNC_ACTION[7]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[7]}" ]; then + if [ "$resumeInitiator" == "${SYNC_ACTION[7]}" ]; then treeList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__treeAfterFile]}" & initiatorPid="$!" fi - if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then + if [ "$resumeTarget" == "${SYNC_ACTION[7]}" ]; then treeList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__treeAfterFile]}" & targetPid="$!" fi @@ -1654,28 +1767,28 @@ function Sync { for pid in "${pidArray[@]}"; do pid=${pid%:*} if [ "$pid" == "$initiatorPid" ]; then - echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__initiatorLastActionFile]}" initiatorFail=true elif [ "$pid" == "$targetPid" ]; then - echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" + echo "${SYNC_ACTION[7]}" > "${INITIATOR[$__targetLastActionFile]}" targetFail=true fi done if [ $initiatorFail == false ]; then - echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__initiatorLastActionFile]}" fi if [ $targetFail == false ]; then - echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" + echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__targetLastActionFile]}" fi exit 1 else - echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__initiatorLastActionFile]}" - echo "${SYNC_ACTION[6]}" > "${INITIATOR[$__targetLastActionFile]}" - resumeInitiator="${SYNC_ACTION[6]}" - resumeTarget="${SYNC_ACTION[6]}" + echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[8]}" + resumeTarget="${SYNC_ACTION[8]}" fi fi @@ -2045,6 +2158,7 @@ function Init { SYNC_ACTION=( 'replica-tree' 'deleted-list' + 'ctime-mtime-list' 'conflict-list' 'sync_attrs' 'update-replica' From a99a96e87164e6d0a4f50ed50930ca910358b1ab Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 14:35:44 +0200 Subject: [PATCH 6/7] WIP: conflict detection --- dev/n_osync.sh | 85 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index cba8861..8cd4cf0 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -856,8 +856,8 @@ ENDSSH function timestampList { local replicaPath="${1}" # path to the replica for which a tree needs to be constructed local replicaType="${2}" # replica type: initiator, target - local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) - local conflictFileName="{4}" + local fileList="${3}" # List of files to get timestamps for + local timestampFilename="${4}" # filename to output timestamp list (will be prefixed with $replicaType) __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG @@ -867,16 +867,33 @@ function timestampList { Logger "Getting file stats for $replicaType replica." "NOTICE" - #_getCtimeMtime - #compare + if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then + timestampFilename="${TARGET[$__type]}" + elif [ "$replicaType" == "${TARGET[$__type]}" ]; then + timestampFilename="${INITIATOR[$__type]}" + else + Logger "Bogus replicaType in [${FUNCNAME[0]}]." "CRITICAL" + exit 1 + fi + Logger "Creating $replicaType replica file list [$replicaPath]." "NOTICE" + if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + _getFileCtimeMtimeRemote "$replicaPath" "$replicaType" "$fileList" "$timestampFilename" + else + _getFileCtimeMtimeLocal "$replicaPath" "$replicaType" "$fileList" "$timestampFilename" + fi } #WIP function conflictList { + return 0 + local replicaPath="${1}" # path to the replica for which a tree needs to be constructed local replicaType="${2}" # replica type: initiator, target - local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) - local conflictFileName="{4}" + local timestampCurrentFilename="${3}" # filename of current timestamp list (will be prefixed with $replicaType) + local timestampPreviousFilename="${3}" # filename of previous timestamp list (will be prefixed with $replicaType) + local conflictFilename="{4}" # filename of conflicts __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG @@ -886,8 +903,19 @@ function conflictList { Logger "Creating conflict list for $replicaType replica." "NOTICE" - #_getCtimeMtime - #compare + # Compare if previous exists +} + +#WIP +function renameConflictList { + return 0 + + local timestampCurrentFilename="${3}" # filename of current timestamp list (will be prefixed with $replicaType) + local timestampPreviousFilename="${3}" # filename of previous timestamp list (will be prefixed with $replicaType) + + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG + + # Rename files } # rsync does sync with mtime, but file attribute modifications only change ctime. @@ -1369,7 +1397,7 @@ function deletionPropagation { fi } -###### Sync function in 6 steps +###### Sync function in 9 steps ###### ###### Step 0a & 0b: Create current file list of replicas ###### Step 1a & 1b: Create deleted file list of replicas @@ -1379,6 +1407,7 @@ function deletionPropagation { ###### Step 5a & 5b: Update replicas ###### Step 6a & 6b: Propagate deletions on replicas ###### Step 7a & 8b: Create after run file list of replicas +###### Step 8: Rename timestamp files from current to previous function Sync { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1535,12 +1564,12 @@ function Sync { ## Step 2a & 2b if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ]; then - timestampList "${INITIATOR[$__type]}" & + timestampList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__type]}${INITIATOR[$__timestampsCurrentFile]}" & initiatorPid="$!" fi if [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then - timestampList "${TARGET[$__type]}" & + timestampList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${TARGET[$__type]}${TARGET[$__timestampsCurrentFile]}" & targetPid="$!" fi @@ -1792,6 +1821,30 @@ function Sync { fi fi + # Step 8 #WIP adapt to last function + if [ "$resumeInitiator" == "${SYNC_ACTION[8]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[8]}" ]; then + if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then + renameTimestampFiles "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false + if [ $? -ne 0 ]; then + echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[8]}" > "${INITIATOR[$__targetLastActionFile]}" + exit 1 + else + echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[5]}" + resumeTarget="${SYNC_ACTION[5]}" + + fi + else + echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__initiatorLastActionFile]}" + echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__targetLastActionFile]}" + resumeInitiator="${SYNC_ACTION[9]}" + resumeTarget="${SYNC_ACTION[9]}" + fi + fi + Logger "Finished synchronization task." "NOTICE" echo "0" > "${INITIATOR[$__resumeCount]}" } @@ -2086,6 +2139,9 @@ function Init { readonly __deletedListFile=13 readonly __failedDeletedListFile=14 readonly __successDeletedListFile=15 + readonly __timestampCurrentFile=15 + readonly __timestampPreviousFile=16 + readonly __conflictListFile=17 INITIATOR=() INITIATOR[$__type]='initiator' @@ -2104,6 +2160,9 @@ function Init { INITIATOR[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" INITIATOR[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" INITIATOR[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" + INITIATOR[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix" + INITIATOR[$__timestampPreviousFile]="-timestamps-previous-$INSTANCE_ID$drySuffix" + INITIATOR[$__conflictListfile]="conflicts-$INSTANCE_ID$drySuffix" TARGET=() TARGET[$__type]='target' @@ -2122,6 +2181,9 @@ function Init { TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" + TARGET[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix" + TARGET[$__timestampPreviousFile]="-timestamps-previous-$INSTANCE_ID$drySuffix" + TARGET[$__conflictListfile]="conflicts-$INSTANCE_ID$drySuffix" PARTIAL_DIR="${INITIATOR[$__partialDir]}" @@ -2164,6 +2226,7 @@ function Init { 'update-replica' 'delete-propagation' 'replica-tree-after' + 'rename-timestamp-files' 'synced' ) } From 5c60e7a9a20820a1f8cddebc802b0180ef824907 Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 30 May 2017 14:36:04 +0200 Subject: [PATCH 7/7] WIP: conflict detection --- dev/n_osync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 8cd4cf0..f21f04b 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2.2-dev -PROGRAM_BUILD=2017053004 +PROGRAM_BUILD=2017053005 IS_STABLE=no