From e019fb98f9d35efe086ee0ddc02c2002d8083213 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 17:54:20 +0200 Subject: [PATCH 1/8] Added missing local declarations --- dev/ofunctions.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 32ae6ee..b69f5e6 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,6 +1,6 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016082801 +## FUNC_BUILD=2016082802 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: @@ -592,7 +592,6 @@ function joinString { # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Warning: Don't imbricate this function into another run if you plan to use the global variable output -#TODO check missing local values used here function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. @@ -613,9 +612,13 @@ function WaitForTaskCompletion { local retval=0 # return value of monitored pid process local errorcount=0 # Number of pids that finished with errors + local pid # Current pid working on local pidCount # number of given pids local pidState # State of the process + local pidsArray # Array of currently running pids + local newPidsArray # New array of currently running pids + IFS=';' read -a pidsArray <<< "$pids" pidCount=${#pidsArray[@]} From c5f6874ad8a202eb47110b8b6bace8e30e31d5ac Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 17:58:03 +0200 Subject: [PATCH 2/8] Added some preflight checks --- dev/n_osync.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index a982ac4..d50bc6c 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash #TODO(critical): handle conflict prevalance, especially in sync_attrs function -#TODO(critical): writelockfiles remote does not shut execution PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016082201 +PROGRAM_BUILD=2016082801 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -159,6 +158,10 @@ function CheckCurrentConfig { done #TODO(low): Add runtime variable tests (RSYNC_ARGS etc) + if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then + Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL" + exit 1 + fi } ###### Osync specific functions (non shared) From 7e51ad54971f2e16372492e8e7b7fac6f59a7793 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 18:22:37 +0200 Subject: [PATCH 3/8] WIP: fixing unlocking depending on lock success --- dev/n_osync.sh | 98 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index d50bc6c..c37a692 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -6,7 +6,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016082801 +PROGRAM_BUILD=2016082802 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -128,6 +128,24 @@ function CheckEnvironment { function CheckCurrentConfig { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + # Check all variables that should contain "yes" or "no" + declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) + for i in "${yes_no_vars[@]}"; do + test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + # Check all variables that should contain a numerical value >= 0 + declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER) + for i in "${num_vars[@]}"; do + test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done +} + +function CheckCurrentConfigAll { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + if [ "$INSTANCE_ID" == "" ]; then Logger "No INSTANCE_ID defined in config file." "CRITICAL" exit 1 @@ -143,20 +161,6 @@ function CheckCurrentConfig { exit 1 fi - # Check all variables that should contain "yes" or "no" - declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) - for i in "${yes_no_vars[@]}"; do - test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" - eval "$test" - done - - # Check all variables that should contain a numerical value >= 0 - declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER) - for i in "${num_vars[@]}"; do - test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" - eval "$test" - done - #TODO(low): Add runtime variable tests (RSYNC_ARGS etc) if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL" @@ -499,6 +503,7 @@ function _WriteLockFilesRemote { cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" + eval "mongo" if [ $? != 0 ]; then Logger "Could not set lock on remote $replica_type replica." "CRITICAL" exit 1 @@ -510,23 +515,38 @@ function _WriteLockFilesRemote { function WriteLockFiles { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local pids + local initiatorPid + local targetPid + local pidArray + local pid _WriteLockFilesLocal "${INITIATOR[2]}" & - pids="$!" + initiatorPid="$!" + if [ "$REMOTE_OPERATION" != "yes" ]; then _WriteLockFilesLocal "${TARGET[2]}" & - pids="$pids;$!" + targetPid="$!" else _WriteLockFilesRemote "${TARGET[2]}" & - pids="$pids;$!" + targetPid="$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING + + WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? -ne 0 ]; then + IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" + for pid in "${pidArray[@]}"; do + pid=${pid%:*} + if [ $pid == $initiatorPid ]; then + INITIATOR_LOCK_FILE_EXISTS=false + elif [ $pid == $targetPid ]; then + TARGET_LOCK_FILE_EXISTS=false + fi + done + Logger "Cancelling task." "CRITICAL" + local exit 1 fi - LOCK_FILES_EXIST=1 } function _UnlockReplicasLocal { @@ -568,21 +588,29 @@ function UnlockReplicas { local pids - if [ $_NOLOCKS -eq 1 ] || [ $LOCK_FILES_EXIST -eq 0 ]; then + if [ $_NOLOCKS -eq 1 ]; then return 0 fi - _UnlockReplicasLocal "${INITIATOR[2]}" & - pids="$!" - - if [ "$REMOTE_OPERATION" != "yes" ]; then - _UnlockReplicasLocal "${TARGET[2]}" & - pids="$pids;$!" - else - _UnlockReplicasRemote "${TARGET[2]}" & - pids="$pids;$!" + if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then + _UnlockReplicasLocal "${INITIATOR[2]}" & + pids="$!" + fi + + #WIP check if WaitForTaskCompletion can handle emppty pids like ";pid" + if [ $TARGET_LOCK_FILE_EXISTS == true ]; then + if [ "$REMOTE_OPERATION" != "yes" ]; then + _UnlockReplicasLocal "${TARGET[2]}" & + pids="$pids;$!" + else + _UnlockReplicasRemote "${TARGET[2]}" & + pids="$pids;$!" + fi + fi + + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING } ###### Sync core functions @@ -1817,7 +1845,10 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then CONFLICT_PREVALANCE=initiator fi -LOCK_FILES_EXIST=0 +# Always assume lock files exist and must be unlocked, unless WriteLockFiles says otherwise +INITIATOR_LOCK_FILE_EXISTS=true +TARGET_LOCK_FILE_EXISTS=true + FORCE_UNLOCK=0 no_maxtime=0 opts="" @@ -1975,6 +2006,7 @@ opts="${opts# *}" if [ $_QUICK_SYNC -lt 2 ]; then CheckCurrentConfig fi + CheckCurrentConfigAll DATE=$(date) Logger "-------------------------------------------------------------" "NOTICE" From 7190ffeae5c9e1e68af9684a44deca63eaeb2032 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 18:26:12 +0200 Subject: [PATCH 4/8] Indented function chronology description --- dev/n_osync.sh | 67 +++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index c37a692..f7f5b75 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -9,40 +9,41 @@ PROGRAM_VERSION=1.2-dev-parallel PROGRAM_BUILD=2016082802 IS_STABLE=no -# Function Name Is parallel #__WITH_PARANOIA_DEBUG +# Execution order +# Function Name Is parallel #__WITH_PARANOIA_DEBUG -# GetLocalOS no #__WITH_PARANOIA_DEBUG -# InitLocalOSSettings no #__WITH_PARANOIA_DEBUG -# CheckEnvironment no #__WITH_PARANOIA_DEBUG -# PreInit no #__WITH_PARANOIA_DEBUG -# Init no #__WITH_PARANOIA_DEBUG -# PostInit no #__WITH_PARANOIA_DEBUG -# GetRemoteOS no #__WITH_PARANOIA_DEBUG -# InitRemoteOSSettings no #__WITH_PARANOIA_DEBUG -# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG -# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG -# RunBeforeHook yes #__WITH_PARANOIA_DEBUG -# Main no #__WITH_PARANOIA_DEBUG -# CreateStateDirs yes #__WITH_PARANOIA_DEBUG -# CheckLocks yes #__WITH_PARANOIA_DEBUG -# WriteLockFiles yes #__WITH_PARANOIA_DEBUG -# Sync no #__WITH_PARANOIA_DEBUG -# tree_list yes #__WITH_PARANOIA_DEBUG -# tree_list yes #__WITH_PARANOIA_DEBUG -# delete_list yes #__WITH_PARANOIA_DEBUG -# delete_list yes #__WITH_PARANOIA_DEBUG -# sync_attrs no #__WITH_PARANOIA_DEBUG -# _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG -# sync_update yes #__WITH_PARANOIA_DEBUG -# sync_update yes #__WITH_PARANOIA_DEBUG -# deletion_propagation yes #__WITH_PARANOIA_DEBUG -# deletion_propagation yes #__WITH_PARANOIA_DEBUG -# tree_list yes #__WITH_PARANOIA_DEBUG -# tree_list yes #__WITH_PARANOIA_DEBUG -# SoftDelete yes #__WITH_PARANOIA_DEBUG -# RunAfterHook yes #__WITH_PARANOIA_DEBUG -# UnlockReplicas yes #__WITH_PARANOIA_DEBUG -# CleanUp no #__WITH_PARANOIA_DEBUG +# GetLocalOS no #__WITH_PARANOIA_DEBUG +# InitLocalOSSettings no #__WITH_PARANOIA_DEBUG +# CheckEnvironment no #__WITH_PARANOIA_DEBUG +# PreInit no #__WITH_PARANOIA_DEBUG +# Init no #__WITH_PARANOIA_DEBUG +# PostInit no #__WITH_PARANOIA_DEBUG +# GetRemoteOS no #__WITH_PARANOIA_DEBUG +# InitRemoteOSSettings no #__WITH_PARANOIA_DEBUG +# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG +# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG +# RunBeforeHook yes #__WITH_PARANOIA_DEBUG +# Main no #__WITH_PARANOIA_DEBUG +# CreateStateDirs yes #__WITH_PARANOIA_DEBUG +# CheckLocks yes #__WITH_PARANOIA_DEBUG +# WriteLockFiles yes #__WITH_PARANOIA_DEBUG +# Sync no #__WITH_PARANOIA_DEBUG +# tree_list yes #__WITH_PARANOIA_DEBUG +# tree_list yes #__WITH_PARANOIA_DEBUG +# delete_list yes #__WITH_PARANOIA_DEBUG +# delete_list yes #__WITH_PARANOIA_DEBUG +# sync_attrs no #__WITH_PARANOIA_DEBUG +# _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG +# sync_update yes #__WITH_PARANOIA_DEBUG +# sync_update yes #__WITH_PARANOIA_DEBUG +# deletion_propagation yes #__WITH_PARANOIA_DEBUG +# deletion_propagation yes #__WITH_PARANOIA_DEBUG +# tree_list yes #__WITH_PARANOIA_DEBUG +# tree_list yes #__WITH_PARANOIA_DEBUG +# SoftDelete yes #__WITH_PARANOIA_DEBUG +# RunAfterHook yes #__WITH_PARANOIA_DEBUG +# UnlockReplicas yes #__WITH_PARANOIA_DEBUG +# CleanUp no #__WITH_PARANOIA_DEBUG source "./ofunctions.sh" _LOGGER_PREFIX="time" From 52b31f7bcac3491a932009e451fb2f8eb34f8d6a Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 18:40:39 +0200 Subject: [PATCH 5/8] Fixed locking / unlocking mechanism --- dev/n_osync.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index f7f5b75..6358b16 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -481,20 +481,22 @@ function CheckLocks { function _WriteLockFilesLocal { local lockfile="${1}" - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local replica_type="${2}" + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" if [ $? != 0 ]; then - Logger "Could not create lock file [$lockfile]." "CRITICAL" + Logger "Could not create lock file on local $replica_type in [$lockfile]." "CRITICAL" exit 1 else - Logger "Locked replica on [$lockfile]." "DEBUG" + Logger "Locked local $replica_type replica in [$lockfile]." "DEBUG" fi } function _WriteLockFilesRemote { local lockfile="${1}" - __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local replica_type="${2}" + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local cmd= @@ -504,12 +506,11 @@ function _WriteLockFilesRemote { cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" - eval "mongo" if [ $? != 0 ]; then - Logger "Could not set lock on remote $replica_type replica." "CRITICAL" + Logger "Could not create lock file on remote $replica_type in [$lockfile]." "CRITICAL" exit 1 else - Logger "Locked remote $replica_type replica." "DEBUG" + Logger "Locked remote $replica_type replica in [$lockfile]." "DEBUG" fi } @@ -521,14 +522,14 @@ function WriteLockFiles { local pidArray local pid - _WriteLockFilesLocal "${INITIATOR[2]}" & + _WriteLockFilesLocal "${INITIATOR[2]}" "${INITIATOR[0]}"& initiatorPid="$!" if [ "$REMOTE_OPERATION" != "yes" ]; then - _WriteLockFilesLocal "${TARGET[2]}" & + _WriteLockFilesLocal "${TARGET[2]}" "${TARGET[0]}" & targetPid="$!" else - _WriteLockFilesRemote "${TARGET[2]}" & + _WriteLockFilesRemote "${TARGET[2]}" "${TARGET[0]}" & targetPid="$!" fi @@ -545,7 +546,6 @@ function WriteLockFiles { done Logger "Cancelling task." "CRITICAL" - local exit 1 fi } From 0aa2d4967846540488b97f465828ffd9dbd9a23c Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 21:07:40 +0200 Subject: [PATCH 6/8] Another lock/unlock fix --- dev/n_osync.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 6358b16..807c374 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -6,7 +6,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016082802 +PROGRAM_BUILD=2016082803 IS_STABLE=no # Execution order @@ -498,7 +498,7 @@ function _WriteLockFilesRemote { local replica_type="${2}" __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local cmd= + local cmd CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -533,6 +533,8 @@ function WriteLockFiles { targetPid="$!" fi + INITIATOR_LOCK_FILE_EXISTS=true + TARGET_LOCK_FILE_EXISTS=true WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? -ne 0 ]; then IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" @@ -1846,9 +1848,8 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then CONFLICT_PREVALANCE=initiator fi -# Always assume lock files exist and must be unlocked, unless WriteLockFiles says otherwise -INITIATOR_LOCK_FILE_EXISTS=true -TARGET_LOCK_FILE_EXISTS=true +INITIATOR_LOCK_FILE_EXISTS=false +TARGET_LOCK_FILE_EXISTS=false FORCE_UNLOCK=0 no_maxtime=0 From edfb092ff72abedd7e70b0df9af25d9851a6aa9d Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 21:17:13 +0200 Subject: [PATCH 7/8] Removed elder comments --- dev/n_osync.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 807c374..6b252f1 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -633,11 +633,6 @@ function tree_list { __CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - #WIP - #if [ "$replica_type" == "${INITIATOR[0]}" ]; then - # exit 12 - #fi - escaped_replica_path=$(EscapeSpaces "$replica_path") Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE" @@ -754,7 +749,6 @@ function sync_attrs { Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" eval "$rsync_cmd" WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING - #WIP: return retval from process instead of err count if only one pid is tested retval=$? if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" @@ -835,7 +829,6 @@ function sync_attrs { Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" eval "$rsync_cmd" WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING - #WIP: the same retval=$? if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE" From 7294fa884698e4e22c7f3a3f1fa5a1fa94eab571 Mon Sep 17 00:00:00 2001 From: deajan Date: Sun, 28 Aug 2016 21:39:09 +0200 Subject: [PATCH 8/8] Updated common merge from other projects --- dev/merge.sh | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/dev/merge.sh b/dev/merge.sh index e59a607..dcac55e 100755 --- a/dev/merge.sh +++ b/dev/merge.sh @@ -1,26 +1,38 @@ #!/usr/bin/env bash -## Merges ofunctions.sh and $PROGRAM +## MERGE 2016080601 + +## Merges ofunctions.sh and n_program.sh into program.sh +## Adds installer PROGRAM=osync VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh) VERSION=${VERSION#*=} -FUNC_PATH=/home/git/common -PARANOIA_DEBUG_LINE="#__WITH_PARANOIA_DEBUG" +PARANOIA_DEBUG_LINE="__WITH_PARANOIA_DEBUG" PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG" PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG" +MINIMUM_FUNCTION_BEGIN="#### MINIMAL-FUNCTION-SET BEGIN ####" +MINIMUM_FUNCTION_END="#### MINIMAL-FUNCTION-SET END ####" function Unexpand { - unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh + unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh } -function Merge { +function MergeAll { - sed "/source \"\.\/ofunctions.sh\"/r /home/git/common/ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh + sed "/source \"\.\/ofunctions.sh\"/r ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh + chmod +x debug_$PROGRAM.sh +} + +function MergeMinimum { + sed -n "/$MINIMUM_FUNCTION_BEGIN/,/$MINIMUM_FUNCTION_END/p" ofunctions.sh > tmp_minimal.sh + sed "/source \"\.\/ofunctions.sh\"/r tmp_minimal.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' | grep -v "$PARANOIA_DEBUG_LINE" > debug_$PROGRAM.sh + rm -f tmp_minimal.sh chmod +x debug_$PROGRAM.sh } + function CleanDebug { # sed explanation @@ -38,16 +50,22 @@ function CleanDebug { } function CopyCommons { - sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_install.sh > ../tmp_install.sh - sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh - sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_batch.sh > ../$PROGRAM-batch.sh - chmod +x ../install.sh - chmod +x ../$PROGRAM-batch.sh + sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../tmp_install.sh + sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh + if [ -f "common_batch.sh" ]; then + sed "s/\[prgname\]/$PROGRAM/g" common_batch.sh > ../$PROGRAM-batch.sh + fi + chmod +x ../install.sh + chmod +x ../$PROGRAM-batch.sh + rm -f ../tmp_install.sh } Unexpand -Merge +if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then + MergeAll +else + MergeMinimum +fi CleanDebug -rm -f tmp_$PROGRAM.sh -rm -f ../tmp_install.sh CopyCommons +rm -f tmp_$PROGRAM.sh