WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changed to more clear WaitForTaskCompletion & ParallelExec argument list

This commit is contained in:
deajan 2016-11-24 09:40:33 +01:00
parent 3a3e7cfa02
commit a0fd7941c7
1 changed files with 38 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
dev/ofunctions.sh
View File

@ -1,6 +1,6 @@
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016112206 ## FUNC_BUILD=2016112401
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -15,9 +15,6 @@
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile
#TODO: Windows checks, check sendmail & mailsend
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2" echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127 exit 127
@ -51,10 +48,10 @@ fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys SLEEP_TIME=.05
_LOGGER_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05 SLEEP_TIME=.05
fi fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
@ -571,23 +568,21 @@ function joinString {
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
# Warning: Don't imbricate this function into another run if you plan to use the global variable output # Warning: Don't imbricate this function into another run if you plan to use the global variable output
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 ${FUNCNAME[0]} true 0 true false # Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true "${FUNCNAME[0]}"
#TODO: function WaitForTask $pid $softTime $hardTime $sleepTime $keepLogging $counting $spinner $noErrror $callerName
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${4}" # Who called this function local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${5:-0}" # Every keepLogging seconds, an 'alive' log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local noError="${7:-false}" # When set to true, reaching soft or hard max time does not trigger an error local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local spinner="${8:-true}" # Do we show spinner ? local noError="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local sleepTime="${9:-0}" # Override SLEEP_TIME local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 6-8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 9 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
@ -612,11 +607,6 @@ function WaitForTaskCompletion {
WAIT_FOR_TASK_COMPLETION="" WAIT_FOR_TASK_COMPLETION=""
# Default value when not overriden
if [ $sleepTime -eq 0 ]; then
sleepTime=$SLEEP_TIME
fi
while [ ${#pidsArray[@]} -gt 0 ]; do while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=() newPidsArray=()
@ -711,15 +701,22 @@ function WaitForTaskCompletion {
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands # Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
#TODO: function ParallelExec $numberOfProcesses $commandsArg $readFromFile $softTime $HardTime $sleepTime $keepLogging $counting $Spinner $noError $callerName
#WIP
function ParallelExec { function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ? local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" local softMaxTime="${4:-0}" #
local hardMaxTime="${5:-0}" local hardMaxTime="${5:-0}"
local callerName="${6}" # Who called this function local sleepTime="${6:-.05}"
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${7:-0}"
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noError="${10:-false}"
local callerName="${11}"
__CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -752,9 +749,12 @@ function ParallelExec {
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then if [ $readFromFile == true ]; then
#TODO: Checked on FreeBSD 10, also check on Win
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else else
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
@ -1105,7 +1105,8 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1140,7 +1141,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1174,7 +1175,7 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1193,7 +1194,7 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1206,7 +1207,7 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1229,7 +1230,7 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"