WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merged improvements from burp branch

This commit is contained in:
deajan 2018-02-20 22:14:41 +01:00
parent 57cf91e4c6
commit 686a28cf08
1 changed files with 439 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

567
dev/ofunctions.sh
View File

@ -2,6 +2,13 @@
#### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS FULL SUBSET ####
#### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MINI SUBSET ####
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling"
#done: add checkRFC function (and use it for --destination-mails)
#done: ExecTasks still needs some better call argument list
#done: ExecTasks sub function relocate
#done: SendMail and SendEmail convert functions inverted, check on osync and obackup
#command line arguments don't take -AaqV for example
_OFUNCTIONS_VERSION=2.2.0-dev _OFUNCTIONS_VERSION=2.2.0-dev
_OFUNCTIONS_BUILD=2018013101 _OFUNCTIONS_BUILD=2018013101
#### _OFUNCTIONS_BOOTSTRAP SUBSET #### #### _OFUNCTIONS_BOOTSTRAP SUBSET ####
@ -410,14 +417,8 @@ function SendAlert {
else else
attachment=true attachment=true
fi fi
if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" ]; then
if [ "$MAIL_BODY_CHARSET" != "" ] && type iconv > /dev/null 2>&1; then body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of log"$'\n\n'"$(tail -n 1000 $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)"
iconv -f UTF-8 -t $MAIL_BODY_CHARSET "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP"
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP)"
else
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)"
fi
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -476,6 +477,24 @@ function SendEmail {
local encryption_string= local encryption_string=
local auth_string= local auth_string=
local i
for i in "${destinationMails}"; do
if [ $(CheckRFC822 "$i") -ne 1 ]; then
Logger "Given email [$i] does not seem to be valid." "WARN"
fi
done
# Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then
if type iconv > /dev/null 2>&1; then
echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP"
message="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP)"
else
Logger "iconv utility not installed. Will not convert email charset." "NOTICE"
fi
fi
if [ ! -f "$attachment" ]; then if [ ! -f "$attachment" ]; then
attachment_command="-a $attachment" attachment_command="-a $attachment"
mail_no_attachment=1 mail_no_attachment=1
@ -666,7 +685,6 @@ function Spinner {
return 0 return 0
else else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER" printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
#printf "\b\b\b\b\b\b"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???} _OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0 return 0
fi fi
@ -684,7 +702,7 @@ function WaitForTaskCompletion {
local noErrorLog="${8:-false}" local noErrorLog="${8:-false}"
local id="${9-base}" local id="${9-base}"
ExecTasks "$id" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorlog" false 1 "$pids" "" "" ExecTasks "$pids" "$id" false 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorlog"
} }
# ParallelExec function emulation, now uses ExecTasks # ParallelExec function emulation, now uses ExecTasks
@ -701,99 +719,106 @@ function ParallelExec {
local noErrorLog="${10:-false}" local noErrorLog="${10:-false}"
if [ $readFromFile == true ]; then if [ $readFromFile == true ]; then
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 6 "$commandsArg" "" "$numberOfProcesses" ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses"
else else
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 3 "$commandsArg" "" "$numberOfProcesses" ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses"
fi fi
} }
## Main asynchronous execution function ## Main asynchronous execution function
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes. ## Function can work in:
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached. ## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for
## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed.
## Function has 8 execution modes ## Example of improved wait $!
## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false
## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
## sleep 15 &
## pid=$!
## sleep 20 &
## pid2=$!
## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes. ## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor ## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line ## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions"
# Example of improved wait $! emulation ## Bear in mind that given commands and conditions need to be quoted
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
# sleep 15 &
# pid=$!
# sleep 20 &
# pid2=$!
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached. ## ExecTasks has the following ofunctions subfunction requirements:
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
# ParallelExecMode also creates output for commands in "$RUN_DIR/$PROGRAM.ExecTasks.$id.$SCRIPT_PID.$TSTAMP"
## ofunctions.sh subfunction requirements:
## Spinner ## Spinner
## Logger ## Logger
## JoinString ## JoinString
## KillChilds ## KillChilds
## Full call
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
function ExecTasks { function ExecTasks {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id # Mandatory arguments
#TODO: not implemented yet local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}" # Optional arguments
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1). local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging. local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false) local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
local spinner="${9:-true}" # Show spinner (true), do not show anything (false) local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true) local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
#TODO not implemented local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
local noErrorLogAtAll="${11:-false}" # Do not log errros at all (false) local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
local execTasksMode="${12:-1}" # In which mode the function should work, see above local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
# Parallelism specific arguments
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i local i
Logger "${FUNCNAME[0]} called in $execTasksMode mode by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 13-15 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll) declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll)
for i in "${num_vars[@]}"; do for i in "${booleans[@]}"; do
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test" eval "$test"
done done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses) declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries)
for i in "${integers[@]}"; do for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test" eval "$test"
done done
fi fi
# Change '-' to '_' in task id
id="${id/-/_}"
# Expand validExitCodes into array
IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
# ParallelExec specific variables # ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid # Array containing pids of commands currently run local commandsArrayPid=() # Array containing pids of commands currently run
local postPoneIfConditionFails # Boolean to check if command needs to be postponed if condition command fails local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed
local temp
# Common variables # Common variables
local pid # Current pid working on local pid # Current pid working on
@ -807,6 +832,7 @@ function ExecTasks {
local exec_time=0 # Seconds since the beginning of this function local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration local newPidsArray # New array of currently running pids for next iteration
@ -827,66 +853,25 @@ function ExecTasks {
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\"" eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false" eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
case $execTasksMode in # Init function variables depending on mode
1)
if [ $numberOfProcesses -gt 0 ]; then
functionMode=ParallelExec
else
functionMode=WaitForTaskCompletion
fi
if [ $readFromFile == false ]; then
if [ $functionMode == "WaitForTaskCompletion" ]; then
IFS=';' read -r -a pidsArray <<< "$mainInput" IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount=${#pidsArray[@]} mainItemCount="${#pidsArray[@]}"
readFromFile=false
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 1 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
2)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else else
Logger "Cannot read file [$mainInput]." "WARN" IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount="${#commandsArray[@]}"
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount="${#commandsConditionArray[@]}"
fi fi
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 2 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
3)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
readFromFile=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 3 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
4)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 4 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
5)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 5 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
6)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else else
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 6 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
7)
if [ -f "$mainInput" ]; then if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput") mainItemCount=$(wc -l < "$mainInput")
readFromFile=true readFromFile=true
@ -898,37 +883,17 @@ function ExecTasks {
else else
Logger "Cannot read file [$auxInput]." "WARN" Logger "Cannot read file [$auxInput]." "WARN"
fi fi
postPoneIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 7 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
8)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
Logger "Cannot read file [$mainInput]." "WARN"
fi fi
if [ -f "$auxInput" ]; then
auxItemCount=$(wc -l < "$auxInput") if [ $functionMode == "WaitForTaskCompletion" ]; then
else # Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode
Logger "Cannot read file [$auxInput]." "WARN" counter=$mainItemCount
fi fi
postPoneIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 8 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
*)
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
exit 1
esac
Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
#while [ ${#pidsArray[@]} -gt 0 ]; do # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
# The counter -lt mainItemCount has to be false for WaitForTaskCompletion modes function _ExecTasksTimeCheck {
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ]; do
newPidsArray=()
if [ $spinner == true ]; then if [ $spinner == true ]; then
Spinner Spinner
fi fi
@ -939,13 +904,13 @@ function ExecTasks {
fi fi
if [ $keepLogging -ne 0 ]; then if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi fi
fi fi
fi fi
@ -953,7 +918,7 @@ function ExecTasks {
if [ $exec_time -gt $softMaxTime ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
softAlert=true softAlert=true
SendAlert true SendAlert true
fi fi
@ -961,15 +926,17 @@ function ExecTasks {
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noTimeErrorLog != true ]; then if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi fi
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
KillChilds $pid true KillChilds $pid true
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE" Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else else
if [ $noErrorLogsAtAll != true ]; then
Logger "Could not stop task with pid [$pid]." "ERROR" Logger "Could not stop task with pid [$pid]." "ERROR"
fi fi
fi
errorcount=$((errorcount+1)) errorcount=$((errorcount+1))
done done
if [ $noTimeErrorLog != true ]; then if [ $noTimeErrorLog != true ]; then
@ -978,70 +945,14 @@ function ExecTasks {
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true" eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount return $errorcount
elif [ $functionMode == "ParallelExec" ]; then else
return $((mainItemCount - counter + ${#pidsArray[@]})) return 129
fi fi
fi fi
}
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them function _ExecTasksPidsCheck {
if [ $functionMode == "ParallelExec" ]; then newPidsArray=()
while [ $counter -lt "$mainItemCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
fi
else
currentCommand="${commandsArray[$counter]}"
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition="${commandsConditionArray[$counter]}"
fi
fi
executeCommand=false
if [ $auxItemCount -ne 0 ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
retval=$?
if [ $retval -ne 0 ]; then
if [ $postPoneIfConditionFails == true ]; then
Logger "Condition not met for command [$currentCommand]. Postponing command." "NOTICE"
if [ $readFromFile == true ]; then
# TODO: we should not write to the original file, but create a copy instead we can write postponed commands to
echo "$currentCommand" >> "$mainInput"
echo "$currentCommandCondition" >> "$auxInput"
else
commandsArray+=($currentCommand)
commandsConditionArray+=($currentCommandCondition)
fi
mainItemCount=$((mainItemCount+1))
# Trivial sleeptime so postponed commands will not stack too fast
sleep $sleepTime
else
Logger "Condition not met for command [$currentCommand]. Ignoring command." "NOTICE"
fi
else
executeCommand=true
fi
else
executeCommand=true
fi
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$currentCommand"
#TODO not implemented
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
else
Logger "Skipping command [$currentCommand]." "DEBUG"
fi
counter=$((counter+1))
done
fi
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then if [ $(IsInteger $pid) -eq 1 ]; then
@ -1049,15 +960,57 @@ function ExecTasks {
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)" pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
#TODO: implement pidsTimeArray[$pid] check here
# Check if pid hasn't run more than soft/hard perProcessTime
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then
if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time [$softPerProcessTime]exceeded for pid [$pid]." "WARN"
if [ "${commandsArrayPid[$pid]}]" != "" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
fi
softAlert=true
SendAlert true
fi
fi
if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then
if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then
Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR"
if [ "${commandsArrayPid[$pid]}]" != "" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
fi
fi
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Command with pid [$pid] stopped successfully." "NOTICE"
else
if [ $noErrorLogsAtAll != true ]; then
Logger "Could not stop command with pid [$pid]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
fi
newPidsArray+=($pid) newPidsArray+=($pid)
fi fi
else else
# pid is dead, get its exit code from wait command # pid is dead, get its exit code from wait command
wait $pid wait $pid
retval=$? retval=$?
if [ $retval -ne 0 ]; then # Check for valid exit codes
Logger "${FUNCNAME[0]} called by [$id] finished monitoring [$pid] [$currentCommad] with exitcode [$retval]." "DEBUG" if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
fi
errorcount=$((errorcount+1)) errorcount=$((errorcount+1))
# Welcome to variable variable bash hell # Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
@ -1066,14 +1019,15 @@ function ExecTasks {
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\"" eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi fi
else else
Logger "${FUNCNAME[0]} called by [$id] finished monitoring [$pid] [$currentCommand] with exitcode [$retval]." "DEBUG" Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
fi fi
fi fi
hasPids=true ##__WITH_PARANOIA_DEBUG hasPids=true ##__WITH_PARANOIA_DEBUG
fi fi
done done
if [ $hasPids == false ]; then ##__WITH_PARANOIA_DEBUG # hasPids can be false on last iteration in ParallelExec mode
if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
@ -1084,16 +1038,145 @@ function ExecTasks {
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG _PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG
}
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do
_ExecTasksTimeCheck
retval=$?
if [ $retval -ne 0 ]; then
return $retval;
fi
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do
_ExecTasksTimeCheck
retval=$?
if [ $retval -ne 0 ]; then
return $retval;
fi
executeCommand=false
isPostponedCommand=false
currentCommand=""
currentCommandCondition=""
needsPostponing=false
if [ $readFromFile == true ]; then
# awk identifies first line as 1 instead of 0 so we need to increase counter
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
fi
# Check if we need to fetch postponed commands
if [ "$currentCommand" == "" ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP")
isPostponedCommand=true
fi
else
currentCommand="${commandsArray[$counter]}"
if [ $auxItemCount -ne 0 ]; then
currentCommandCondition="${commandsConditionArray[$counter]}"
fi
if [ "$currentCommand" == "" ]; then
currentCommand="${postponedCommandsArray[$postponedCounter]}"
currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}"
isPostponedCommand=true
fi
fi
# Check if we execute postponed commands, or if we delay them
if [ $isPostponedCommand == true ]; then
# Get first value before '@'
postponedExecTime="${currentCommand%%@*}"
postponedExecTime=$((SECONDS-postponedExecTime))
# Get everything after first '@'
temp="${currentCommand#*@}"
# Get first value before '@'
postponedRetryCount="${temp%%@*}"
# Replace currentCommand with actual filtered currentCommand
currentCommand="${temp#*@}"
# Since we read a postponed command, we may decrase postponedItemCounter
postponedItemCount=$((postponedItemCount-1))
#Since we read one line, we need to increase the counter
postponedCounter=$((postponedCounter+1))
else
postponedRetryCount=0
postponedExecTime=0
fi
if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $((minTimeBetweenRetries)) ]) || [ $isPostponedCommand == false ]; then
if [ "$currentCommandCondition" != "" ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true
subRetval=$?
if [ $subRetval -ne 0 ]; then
# is postponing enabled ?
if [ $maxPostponeRetries -gt 0 ]; then
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE"
postponedRetryCount=$((postponedRetryCount+1))
if [ $postponedRetryCount -ge $maxPostponeRetries ]; then
Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE"
else
needsPostponing=true
fi
postponedExecTime=0
else
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE"
fi
else
executeCommand=true
fi
else
executeCommand=true
fi
else
needsPostponing=true
fi
if [ $needsPostponing == true ]; then
postponedItemCount=$((postponedItemCount+1))
if [ $readFromFile == true ]; then
echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP"
echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP"
else
postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand")
postponedCommandsConditionArray+=("$currentCommandCondition")
fi
fi
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$currentCommand"
# Initialize pid execution time array
pidsTimeArray[$pid]=0
else
Logger "Skipping command [$currentCommand]." "DEBUG"
fi
if [ $isPostponedCommand == false ]; then
counter=$((counter+1))
fi
_ExecTasksPidsCheck
done
fi
_ExecTasksPidsCheck
done done
Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors # Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
#WIP: return code has nothing to do with logging
#if [ $noErrorLogAtAll == true ]; then
# return 0
#fi
if [ $mainItemCount -eq 1 ]; then if [ $mainItemCount -eq 1 ]; then
return $retval return $retval
@ -1164,6 +1247,20 @@ function IsNumeric {
fi fi
} }
#### CheckRFC822 SUBSET ####
# Checks email address validity
function checkRFC822 {
local mail="${1}"
local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$"
if [[ $mail =~ $rfc822 ]]; then
echo 1
else
echo 0
fi
}
#### CheckRFC822 SUBSET END ####
#### IsInteger SUBSET #### #### IsInteger SUBSET ####
function IsInteger { function IsInteger {
local value="${1}" local value="${1}"
@ -1506,7 +1603,8 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $! ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1547,7 +1645,8 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $! ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1581,7 +1680,8 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi fi
} }
@ -1600,7 +1700,8 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi fi
} }
@ -1613,7 +1714,8 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $! ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1637,7 +1739,8 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $! ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"