WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored wait functions

This commit is contained in:
deajan 2016-08-07 23:54:46 +02:00
parent bef9a92526
commit bf0051c63d
2 changed files with 53 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
dev/n_osync.sh
View File

@ -1,10 +1,12 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Remove Waitfor function in already sent to background functions to reduce code overhead
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-dev-parallel-unstable PROGRAM_VERSION=1.2-dev-parallel-unstable
PROGRAM_BUILD=2016080401 PROGRAM_BUILD=2016080701
IS_STABLE=no IS_STABLE=no
# Function Name Is parallel #__WITH_PARANOIA_DEBUG # Function Name Is parallel #__WITH_PARANOIA_DEBUG
@ -200,7 +202,7 @@ function _CheckReplicaPathsRemote {
cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1' cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Remote replica path [$replica_path] is not writable." "CRITICAL" Logger "Remote replica path [$replica_path] is not writable." "CRITICAL"
exit 1 exit 1
@ -209,7 +211,7 @@ function _CheckReplicaPathsRemote {
cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot create remote replica path [$replica_path]." "CRITICAL" Logger "Cannot create remote replica path [$replica_path]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
@ -241,7 +243,7 @@ function CheckReplicaPaths {
_CheckReplicaPathsRemote "${TARGET[1]}" & _CheckReplicaPathsRemote "${TARGET[1]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
} }
function _CheckDiskSpaceLocal { function _CheckDiskSpaceLocal {
@ -273,7 +275,7 @@ function _CheckDiskSpaceRemote {
cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot get free space on target [$replica_path]." "ERROR" Logger "Cannot get free space on target [$replica_path]." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
@ -304,7 +306,7 @@ function CheckDiskSpace {
_CheckDiskSpaceRemote "${TARGET[1]}" & _CheckDiskSpaceRemote "${TARGET[1]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
} }
@ -334,7 +336,7 @@ function _CreateStateDirsRemote {
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL" Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
@ -356,7 +358,7 @@ function CreateStateDirs {
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" & _CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
} }
function _CheckLocksLocal { function _CheckLocksLocal {
@ -397,7 +399,7 @@ function _CheckLocksRemote {
cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'"' cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'"'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID) lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)
@ -458,7 +460,7 @@ function CheckLocks {
_CheckLocksRemote "${TARGET[2]}" & _CheckLocksRemote "${TARGET[2]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
WriteLockFiles WriteLockFiles
} }
@ -487,7 +489,7 @@ function _WriteLockFilesRemote {
cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not set lock on remote $replica_type replica." "CRITICAL" Logger "Could not set lock on remote $replica_type replica." "CRITICAL"
exit 1 exit 1
@ -510,7 +512,7 @@ function WriteLockFiles {
_WriteLockFilesRemote "${TARGET[2]}" & _WriteLockFilesRemote "${TARGET[2]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
LOCK_FILES_EXIST=1 LOCK_FILES_EXIST=1
} }
@ -540,7 +542,7 @@ function _UnlockReplicasRemote {
cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not unlock remote replica." "ERROR" Logger "Could not unlock remote replica." "ERROR"
Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
@ -568,7 +570,7 @@ function UnlockReplicas {
_UnlockReplicasRemote "${TARGET[2]}" & _UnlockReplicasRemote "${TARGET[2]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
} }
###### Sync core functions ###### Sync core functions
@ -600,9 +602,9 @@ function tree_list {
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE" ## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE"
## When log writing fails, $! is empty and WaitForCompletion fails. Removing the 2>> log ## When log writing fails, $! is empty and WaitForTaskCompletion fails. Removing the 2>> log
eval "$rsync_cmd" eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
## Retval 24 = some files vanished while creating list ## Retval 24 = some files vanished while creating list
if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f "$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID" ]; then if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f "$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID" ]; then
@ -673,7 +675,7 @@ function _get_file_ctime_mtime_remote {
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"' cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG" Logger "CMD: $cmd" "DEBUG"
eval $cmd eval $cmd
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL" Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" ]; then
@ -705,7 +707,7 @@ function sync_attrs {
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
@ -781,7 +783,7 @@ function sync_attrs {
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE" Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
@ -836,7 +838,7 @@ function sync_update {
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID)" "NOTICE" Logger "List:\n$(cat $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID)" "NOTICE"
@ -1072,7 +1074,7 @@ function deletion_propagation {
delete_dir="${INITIATOR[5]}" delete_dir="${INITIATOR[5]}"
_delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" & _delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Deletion on replica $replica_type failed." "CRITICAL" Logger "Deletion on replica $replica_type failed." "CRITICAL"
@ -1087,7 +1089,7 @@ function deletion_propagation {
else else
_delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" & _delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" &
fi fi
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $retval == 0 ]; then if [ $retval == 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE -eq 1 ]; then if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE -eq 1 ]; then
@ -1310,7 +1312,7 @@ function _SoftDeleteLocal {
else else
Dummy & Dummy &
fi fi
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on $replica_type replica." "ERROR" Logger "Error while executing cleanup on $replica_type replica." "ERROR"
@ -1345,7 +1347,7 @@ function _SoftDeleteRemote {
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
@ -1357,7 +1359,7 @@ function _SoftDeleteRemote {
else else
Dummy & Dummy &
fi fi
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote $replica_type replica." "ERROR" Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
@ -1384,7 +1386,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS & _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
fi fi
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
@ -1399,7 +1401,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS & _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
fi fi
} }

95
dev/ofunctions.sh
View File

@ -1,6 +1,6 @@
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016080601 ## FUNC_BUILD=2016080701
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode #TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode
@ -569,6 +569,7 @@ function WaitForTaskCompletion {
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function local caller_name="${4}" # Who called this function
local exit_on_error="${5:-false}" # Should the function exit on subprocess errors local exit_on_error="${5:-false}" # Should the function exit on subprocess errors
local counting="{6:-true}" # Count time since function launch if true, script launch if false
Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 5 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 5 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -603,7 +604,12 @@ function WaitForTaskCompletion {
done done
Spinner Spinner
if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin)) exec_time=$(($SECONDS - $seconds_begin))
else
exec_time=$SECONDS
fi
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time log_ttime=$exec_time
@ -623,10 +629,8 @@ function WaitForTaskCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
#return 0
else else
errrorcount=$((errorcount+1)) errrorcount=$((errorcount+1))
#return 1
fi fi
fi fi
fi fi
@ -644,54 +648,14 @@ function WaitForTaskCompletion {
fi fi
} }
function WaitForCompletion { function CleanUp {
local pid="${1}" # pid to wait for __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once if [ "$_DEBUG" != "yes" ]; then
local log_time=0 # local time instance for comparaison rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
local seconds_begin=$SECONDS # Seconds since the beginning of the script rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.tmp"
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while eval "$PROCESS_TEST_CMD" > /dev/null
do
Spinner
if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_time -ne $SECONDS ]; then
log_time=$SECONDS
Logger "Current task still running." "NOTICE"
fi fi
fi
if [ $SECONDS -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
Logger "Max soft execution time exceeded for script." "WARN"
soft_alert=1
SendAlert
fi
if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
return $retval
} }
#### MINIMAL-FUNCTION-SET END #### #### MINIMAL-FUNCTION-SET END ####
@ -762,16 +726,6 @@ function urlDecode {
printf '%b' "${url_encoded//%/\\x}" printf '%b' "${url_encoded//%/\\x}"
} }
function CleanUp {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.tmp"
fi
}
function GetLocalOS { function GetLocalOS {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -823,19 +777,19 @@ function GetRemoteOS {
cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-1" WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-1" false true
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-2" WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-2" false true
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-3" WaitForTaskCompletion $! 120 240 ${FUNCNAME[0]}"-3" false true
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot Get remote OS type." "ERROR" Logger "Cannot Get remote OS type." "ERROR"
@ -888,7 +842,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} false true
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -923,7 +877,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} false true
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -957,7 +911,7 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false true
fi fi
} }
@ -976,7 +930,7 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false true
fi fi
} }
@ -987,7 +941,7 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 180 ${FUNCNAME[0]} WaitForTaskCompletion $! 180 180 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot ping $REMOTE_HOST" "ERROR" Logger "Cannot ping $REMOTE_HOST" "ERROR"
return 1 return 1
@ -1008,7 +962,7 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 360 360 ${FUNCNAME[0]} WaitForTaskCompletion $! 360 360 ${FUNCNAME[0]} false true
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot ping 3rd party host $i" "NOTICE" Logger "Cannot ping 3rd party host $i" "NOTICE"
else else
@ -1266,14 +1220,9 @@ function InitLocalOSSettings {
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ]; then if [ "$LOCAL_OS" == "msys" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
# PROCESS_TEST_CMD assumes there is a variable $pid
# Tested on MSYS and cygwin
PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid'
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
else else
FIND_CMD=find FIND_CMD=find
# PROCESS_TEST_CMD assumes there is a variable $pid
PROCESS_TEST_CMD='ps -p$pid'
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi