diff --git a/dev/n_osync.sh b/dev/n_osync.sh index db5958c..56b830c 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -1,12 +1,16 @@ #!/usr/bin/env bash #TODO(critical): handle conflict prevalance, especially in sync_attrs function +#TODO(high): test resume system +#TODO(critical): test new WaitForTaskCompletion behavior with self=true +#TODO(low): add keep logging & preserve attr conf entries to preflight check +#TODO(low): test failed _CreateStateDirsRemote should stop osync PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-dev-parallel -PROGRAM_BUILD=2016081301 +PROGRAM_BUILD=2016081802 IS_STABLE=no # Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -61,7 +65,7 @@ function TrapStop { if [ $SOFT_STOP -eq 1 ]; then Logger " /!\ WARNING: CTRL+C hit twice. Exiting osync. Please wait while replicas get unlocked..." "WARN" SOFT_STOP=2 - exit 1 + exit 2 fi } @@ -95,7 +99,7 @@ function TrapQuit { fi CleanUp Logger "$PROGRAM finished with warnings." "WARN" - exitcode=240 # Special exit code for daemon mode not stopping on warnings + exitcode=2 # Warning exit code must not force daemon mode to quit else UnlockReplicas if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then @@ -146,7 +150,7 @@ function CheckCurrentConfig { fi # Check all variables that should contain "yes" or "no" - declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) + declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) for i in "${yes_no_vars[@]}"; do test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" eval "$test" @@ -241,7 +245,11 @@ function CheckReplicaPaths { _CheckReplicaPathsRemote "${TARGET[1]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false $KEEP_LOGGING + if [ $? -ne 0 ]; then + Logger "Cancelling task." "CRITICAL" + exit 1 + fi } function _CheckDiskSpaceLocal { @@ -303,7 +311,7 @@ function CheckDiskSpace { _CheckDiskSpaceRemote "${TARGET[1]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING } @@ -354,7 +362,11 @@ function CreateStateDirs { _CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING + if [ $? -ne 0 ]; then + Logger "Cancelling task." "CRITICAL" + exit 1 + fi } function _CheckLocksLocal { @@ -455,7 +467,11 @@ function CheckLocks { _CheckLocksRemote "${TARGET[2]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING + if [ $? -ne 0 ]; then + Logger "Cancelling task." "CRITICAL" + exit 1 + fi WriteLockFiles } @@ -506,7 +522,11 @@ function WriteLockFiles { _WriteLockFilesRemote "${TARGET[2]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGINg + if [ $? -ne 0 ]; then + Logger "Cancelling task." "CRITICAL" + exit 1 + fi LOCK_FILES_EXIST=1 } @@ -563,7 +583,7 @@ function UnlockReplicas { _UnlockReplicasRemote "${TARGET[2]}" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING } ###### Sync core functions @@ -698,7 +718,8 @@ function sync_attrs { fi Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" eval "$rsync_cmd" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING + #WIP: return retval from process instead of err count if only one pid is tested retval=$? if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" @@ -709,7 +730,7 @@ function sync_attrs { if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi - exit $retval + exit 1 #WIP: check why exit $retval was used else cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" if [ $? != 0 ]; then @@ -730,7 +751,7 @@ function sync_attrs { _get_file_ctime_mtime_remote "${TARGET[1]}" "${TARGET[0]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" & pids="$pids;$!" fi - WaitForTaskCompletion $pids 1800 0 ${FUNCNAME[0]} false true + WaitForTaskCompletion $pids 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING # If target gets updated first, then sync_attr must update initiator's attrs first # For join, remove leading replica paths @@ -778,7 +799,8 @@ function sync_attrs { Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" eval "$rsync_cmd" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING + #WIP: the same retval=$? if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE" @@ -789,7 +811,7 @@ function sync_attrs { if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE" fi - exit $retval + exit 1 #WIP check why exit $retval was used else Logger "Successfully updated file attributes on $dest_replica replica." "NOTICE" fi @@ -843,7 +865,7 @@ function sync_update { if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID)" "NOTICE" fi - exit $retval + exit 1 #WIP exit $retval else Logger "Updating $destination_replica replica succeded." "NOTICE" return 0 @@ -1039,7 +1061,7 @@ ENDSSH if [ -f "$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID" ]; then Logger "Comand output: $(cat $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID)" "NOTICE" fi - exit $result + exit 1 #WIP exit $retval fi return 0 } @@ -1174,7 +1196,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" for pid in "${pidArray[@]}"; do @@ -1206,7 +1228,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" for pid in "${pidArray[@]}"; do @@ -1230,7 +1252,7 @@ function Sync { if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$RSYNC_ATTR_ARGS" != "" ]; then sync_attrs "${INITIATOR[1]}" "$TARGET_SYNC_DIR" - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING if [ $? != 0 ]; then echo "${SYNC_ACTION[2]}" > "${INITIATOR[7]}" echo "${SYNC_ACTION[2]}" > "${INITIATOR[8]}" @@ -1262,7 +1284,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" for pid in "${pidArray[@]}"; do @@ -1294,7 +1316,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" for pid in "${pidArray[@]}"; do @@ -1327,7 +1349,7 @@ function Sync { targetPid="$!" fi - WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false + WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING if [ $? != 0 ]; then IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" for pid in "${pidArray[@]}"; do @@ -1450,7 +1472,7 @@ function SoftDelete { _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING fi if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then @@ -1465,7 +1487,7 @@ function SoftDelete { _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS & pids="$pids;$!" fi - WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true + WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING fi } @@ -1696,7 +1718,7 @@ function SyncOnChanges { Logger "daemon cmd: $cmd" "DEBUG" eval "$cmd" retval=$? - if [ $retval != 0 ] && [ $retval != 240 ]; then + if [ $retval != 0 ] && [ $retval != 2 ]; then Logger "osync child exited with error." "ERROR" fi @@ -1906,5 +1928,4 @@ opts="${opts# *}" if [ $? == 0 ]; then SoftDelete fi - RunAfterHook fi