From 0be2bc3bc3aee28c02e055c253e7e6fe7dcb27a2 Mon Sep 17 00:00:00 2001 From: deajan Date: Wed, 19 Oct 2016 12:28:30 +0200 Subject: [PATCH] Updated ofunctions --- dev/ofunctions.sh | 137 +++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 57 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 8c2db6d..a3a5ab9 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,6 +1,6 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016091601 +## FUNC_BUILD=2016101901 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: @@ -103,7 +103,7 @@ function _Logger { local evalue="${3}" # What to log to stderr echo -e "$lvalue" >> "$LOG_FILE" - CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" #WIP + CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then cat <<< "$evalue" 1>&2 @@ -536,6 +536,7 @@ function TrapError { local job="$0" local line="$1" local code="${2:-1}" + if [ $_SILENT == false ]; then echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" fi @@ -543,6 +544,7 @@ function TrapError { function LoadConfigFile { local configFile="${1}" + __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -606,13 +608,13 @@ function joinString { function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon - local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. - local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. - local caller_name="${4}" # Who called this function + local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. + local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. + local callerName="${4}" # Who called this function local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false - local keep_logging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging + local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging - Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once @@ -648,8 +650,8 @@ function WaitForTaskCompletion { exec_time=$SECONDS fi - if [ $keep_logging -ne 0 ]; then - if [ $((($exec_time + 1) % $keep_logging)) -eq 0 ]; then + if [ $keepLogging -ne 0 ]; then + if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s log_ttime=$exec_time Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" @@ -657,15 +659,15 @@ function WaitForTaskCompletion { fi fi - if [ $exec_time -gt $soft_max_time ]; then - if [ $soft_alert == true ] && [ $soft_max_time -ne 0 ]; then - Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + if [ $exec_time -gt $softMaxTime ]; then + if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then + Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" soft_alert=true SendAlert true fi - if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then - Logger "Max hard execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" for pid in "${pidsArray[@]}"; do KillChilds $pid true if [ $? == 0 ]; then @@ -693,7 +695,7 @@ function WaitForTaskCompletion { retval=$? if [ $retval -ne 0 ]; then errorcount=$((errorcount+1)) - Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG" + Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG" if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then WAIT_FOR_TASK_COMPLETION="$pid:$retval" else @@ -714,7 +716,7 @@ function WaitForTaskCompletion { sleep $SLEEP_TIME done - Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Return exit code if only one process was monitored, else return number of errors if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then @@ -731,8 +733,13 @@ function ParallelExec { local numberOfProcesses="${1}" # Number of simultaneous commands to run local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line local readFromFile="${3:-false}" # Is commandsArg a file or a string ? + local softMaxTime="${4:-0}" + local hardMaxTime="${5:-0}" + local callerName="${6}" # Who called this function + local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false + local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging - __CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local commandCount local command @@ -771,7 +778,7 @@ function ParallelExec { command="${commandsArray[$counter]}" fi Logger "Running command [$command]." "DEBUG" - eval "$command" & + eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & pid=$! pidsArray+=($pid) commandsArrayPid[$pid]="$command" @@ -831,6 +838,7 @@ function SedStripQuotes { # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" + string="${string/#\'/}" # Remove singlequote if it begins string string="${string/%\'/}" # Remove singlequote if it ends string echo "$string" @@ -839,6 +847,7 @@ function StripSingleQuotes { # Usage: var=$(StripDoubleQuotes "$var") function StripDoubleQuotes { local string="${1}" + string="${string/#\"/}" string="${string/%\"/}" echo "$string" @@ -846,12 +855,14 @@ function StripDoubleQuotes { function StripQuotes { local string="${1}" + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" } # Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")" function EscapeSpaces { local string="${1}" # String on which spaces will be escaped + echo "${string// /\\ }" } @@ -859,6 +870,7 @@ function IsNumericExpand { eval "local value=\"${1}\"" # Needed eval so variable variables can be processed local re="^-?[0-9]+([.][0-9]+)?$" + if [[ $value =~ $re ]]; then echo 1 else @@ -905,25 +917,25 @@ function urlEncode { } function urlDecode { - local url_encoded="${1//+/ }" + local urlEncoded="${1//+/ }" - printf '%b' "${url_encoded//%/\\x}" + printf '%b' "${urlEncoded//%/\\x}" } function GetLocalOS { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local local_os_var= + local localOsVar - local_os_var="$(uname -spio 2>&1)" + localOsVar="$(uname -spio 2>&1)" if [ $? != 0 ]; then - local_os_var="$(uname -v 2>&1)" + localOsVar="$(uname -v 2>&1)" if [ $? != 0 ]; then - local_os_var="$(uname)" + localOsVar="$(uname)" fi fi - case $local_os_var in + case $localOsVar in *"Linux"*) LOCAL_OS="Linux" ;; @@ -938,14 +950,14 @@ function GetLocalOS { ;; *) if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option - Logger "Running on unknown local OS [$local_os_var]." "WARN" + Logger "Running on unknown local OS [$localOsVar]." "WARN" return fi - Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" + Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR" exit 1 ;; esac - Logger "Local OS: [$local_os_var]." "DEBUG" + Logger "Local OS: [$localOsVar]." "DEBUG" } #### MINIMAL-FUNCTION-SET END #### @@ -953,8 +965,8 @@ function GetLocalOS { function GetRemoteOS { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local cmd= - local remote_os_var= + local cmd + local remoteOsVar if [ "$REMOTE_OPERATION" == "yes" ]; then @@ -983,9 +995,9 @@ function GetRemoteOS { fi fi - remote_os_var=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") + remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") - case $remote_os_var in + case $remoteOsVar in *"Linux"*) REMOTE_OS="Linux" ;; @@ -1004,21 +1016,21 @@ function GetRemoteOS { ;; *) if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option - Logger "Running on unknown remote OS [$remote_os_var]." "WARN" + Logger "Running on unknown remote OS [$remoteOsVar]." "WARN" return fi Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" - Logger "Remote OS said:\n$remote_os_var" "CRITICAL" + Logger "Remote OS said:\n$remoteOsVar" "CRITICAL" exit 1 esac - Logger "Remote OS: [$remote_os_var]." "DEBUG" + Logger "Remote OS: [$remoteOsVar]." "DEBUG" fi } function RunLocalCommand { local command="${1}" # Command to run - local hard_max_time="${2}" # Max time to wait for command to compleet + local hardMaxTime="${2}" # Max time to wait for command to compleet __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG if [ $_DRYRUN == true ]; then @@ -1028,7 +1040,7 @@ function RunLocalCommand { Logger "Running command [$command] on local host." "NOTICE" eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & - WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ $retval -eq 0 ]; then Logger "Command succeded." "NOTICE" @@ -1049,7 +1061,7 @@ function RunLocalCommand { ## Runs remote command $1 and waits for completition in $2 seconds function RunRemoteCommand { local command="${1}" # Command to run - local hard_max_time="${2}" # Max time to wait for command to compleet + local hardMaxTime="${2}" # Max time to wait for command to compleet __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG CheckConnectivity3rdPartyHosts @@ -1063,7 +1075,7 @@ function RunRemoteCommand { cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ $retval -eq 0 ]; then Logger "Command succeded." "NOTICE" @@ -1085,7 +1097,7 @@ function RunRemoteCommand { function RunBeforeHook { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local pids= + local pids if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & @@ -1142,13 +1154,13 @@ function CheckConnectivityRemoteHost { function CheckConnectivity3rdPartyHosts { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local remote_3rd_party_success + local remote3rdPartySuccess local retval if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then - remote_3rd_party_success=false + remote3rdPartySuccess=false for i in $REMOTE_3RD_PARTY_HOSTS do eval "$PING_CMD $i > /dev/null 2>&1" & @@ -1157,11 +1169,11 @@ function CheckConnectivity3rdPartyHosts { if [ $retval != 0 ]; then Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" else - remote_3rd_party_success=true + remote3rdPartySuccess=true fi done - if [ $remote_3rd_party_success == false ]; then + if [ $remote3rdPartySuccess == false ]; then Logger "No remote 3rd party host responded to ping. No internet ?" "WARN" return 1 else @@ -1224,7 +1236,7 @@ function __CheckArguments { #__END_WITH_PARANOIA_DEBUG function RsyncPatternsAdd { - local pattern_type="${1}" # exclude or include + local patternType="${1}" # exclude or include local pattern="${2}" __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -1236,7 +1248,7 @@ function RsyncPatternsAdd { while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR - str=${rest%%;*} + str=${rest%%;*} #TODO: replace ; with $PATH_SEPARATOR_CHAR # Handle the last case if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then rest= @@ -1245,26 +1257,26 @@ function RsyncPatternsAdd { rest=${rest#*$PATH_SEPARATOR_CHAR} fi if [ "$RSYNC_PATTERNS" == "" ]; then - RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" + RSYNC_PATTERNS="--"$patternType"=\"$str\"" else - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\"" fi done set +f } function RsyncPatternsFromAdd { - local pattern_type="${1}" - local pattern_from="${2}" + local patternType="${1}" + local patternFrom="${2}" __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG ## Check if the exclude list has a full path, and if not, add the config file path if there is one - if [ "$(basename $pattern_from)" == "$pattern_from" ]; then - pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" + if [ "$(basename $patternFrom)" == "$patternFrom" ]; then + patternFrom="$(dirname $CONFIG_FILE)/$patternFrom" fi - if [ -e "$pattern_from" ]; then - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" + if [ -e "$patternFrom" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\"" fi } @@ -1428,9 +1440,20 @@ function PostInit { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG # Define remote commands - SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" + if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then + SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" + elif [ -f "$SSH_PASSWORD_FILE" ]; then + SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT" + else + SSH_PASSWORD="" + SSH_CMD="" + SCP_CMD="" + RSYNC_SSH_CMD="" + fi } function InitLocalOSSettings {