WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2018-08-08 11:31:58 +02:00
parent 4cf697cd28
commit 1eec242cb0
5 changed files with 466 additions and 435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

299
dev/debug_osync.sh
View File

@ -5,15 +5,12 @@
#Check dryruns with nosuffix mode for timestampList #Check dryruns with nosuffix mode for timestampList
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong" AUTHOR="(C) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.5-dev PROGRAM_VERSION=1.3.0-beta1
PROGRAM_BUILD=2018062506 PROGRAM_BUILD=2018080801
IS_STABLE=no IS_STABLE=no
#TODO: tidy up ExecTasks comments
##### Execution order #__WITH_PARANOIA_DEBUG ##### Execution order #__WITH_PARANOIA_DEBUG
##### Function Name Is parallel #__WITH_PARANOIA_DEBUG ##### Function Name Is parallel #__WITH_PARANOIA_DEBUG
# GetLocalOS no #__WITH_PARANOIA_DEBUG # GetLocalOS no #__WITH_PARANOIA_DEBUG
@ -44,16 +41,10 @@ IS_STABLE=no
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG # UnlockReplicas yes #__WITH_PARANOIA_DEBUG
# CleanUp no #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.3.0-beta1
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" _OFUNCTIONS_BUILD=2018080801
#command line arguments don't take -AaqV for example
_OFUNCTIONS_VERSION=2.3.0-dev
_OFUNCTIONS_BUILD=2018062504
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
## PROGRAM=program-name ## PROGRAM=program-name
## INSTANCE_ID=program-instance-name ## INSTANCE_ID=program-instance-name
@ -63,8 +54,6 @@ _OFUNCTIONS_BOOTSTRAP=true
## _LOGGER_ERR_ONLY=true/false ## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/"" ## _LOGGER_PREFIX="date"/"time"/""
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP ## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
@ -252,7 +241,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -274,7 +263,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -389,7 +378,6 @@ function KillChilds {
fi fi
if kill -0 "$pid" > /dev/null 2>&1; then if kill -0 "$pid" > /dev/null 2>&1; then
#TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL" Logger "Bogus pgrep implementation." "CRITICAL"
@ -511,6 +499,12 @@ function SendAlert {
# encryption can be set to tls, ssl or none # encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional # smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"
# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc
#set ttycharset=iso-8859-1
#set sendcharsets=iso-8859-1
#set encoding=8bit
function SendEmail { function SendEmail {
local subject="${1}" local subject="${1}"
local message="${2}" local message="${2}"
@ -533,11 +527,16 @@ function SendEmail {
local i local i
for i in "${destinationMails}"; do if [ "${destinationMails[@]}" != "" ]; then
if [ $(CheckRFC822 "$i") -ne 1 ]; then for i in "${destinationMails[@]}"; do
Logger "Given email [$i] does not seem to be valid." "WARN" if [ $(CheckRFC822 "$i") -ne 1 ]; then
fi Logger "Given email [$i] does not seem to be valid." "WARN"
done fi
done
else
Logger "No valid email adresses given." "WARN"
return 1
fi
# Prior to sending an email, convert its body if needed # Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then if [ "$MAIL_BODY_CHARSET" != "" ]; then
@ -809,33 +808,33 @@ function ParallelExec {
function ExecTasks { function ExecTasks {
# Mandatory arguments # Mandatory arguments
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments # Optional arguments
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments # Parallelism specific arguments
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i local i
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -858,40 +857,40 @@ function ExecTasks {
IFS=';' read -r -a validExitCodes <<< "$validExitCodes" IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
# ParallelExec specific variables # ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing pids of commands currently run local commandsArrayPid=() # Array containing pids of commands currently run
local postponedRetryCount=0 # Number of current postponed commands retries local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0 local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ? local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed local needsPostponing # Does currentCommand need to be postponed
local temp local temp
# Common variables # Common variables
local pid # Current pid working on local pid # Current pid working on
local pidState # State of the process local pidState # State of the process
local mainItemCount=0 # number of given items (pids or commands) local mainItemCount=0 # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true) local readFromFile # Should we read pids / commands from a file (true)
local counter=0 local counter=0
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
local functionMode local functionMode
@ -944,7 +943,7 @@ function ExecTasks {
counter=$mainItemCount counter=$mainItemCount
fi fi
Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
function _ExecTasksTimeCheck { function _ExecTasksTimeCheck {
@ -1056,11 +1055,11 @@ function ExecTasks {
else else
# pid is dead, get its exit code from wait command # pid is dead, get its exit code from wait command
wait $pid wait $pid
retval=$? #TODO: do we use retval codes somehow ?? where retval=$?
# Check for valid exit codes # Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
if [ "$functionMode" == "ParallelExec" ]; then if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi fi
@ -1076,22 +1075,22 @@ function ExecTasks {
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
fi fi
fi fi
hasPids=true ##__WITH_PARANOIA_DEBUG hasPids=true ##__WITH_PARANOIA_DEBUG
fi fi
done done
# hasPids can be false on last iteration in ParallelExec mode # hasPids can be false on last iteration in ParallelExec mode
if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $sleepTime sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG _PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG
} }
@ -1280,6 +1279,13 @@ function EscapeSpaces {
echo "${string// /\\ }" echo "${string// /\\ }"
} }
# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")"
function EscapeDoubleQuotes {
local value="${1}"
echo "${value//\"/\\\"}"
}
function IsNumericExpand { function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
@ -1368,6 +1374,8 @@ function CheckRFC822 {
function UrlEncode { function UrlEncode {
local length="${#1}" local length="${#1}"
local i
local LANG=C local LANG=C
for i in $(seq 0 $((length-1))); do for i in $(seq 0 $((length-1))); do
local c="${1:i:1}" local c="${1:i:1}"
@ -1644,6 +1652,9 @@ GetOs
ENDSSH ENDSSH
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL" Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
fi
exit 1 exit 1
fi fi
@ -2062,7 +2073,7 @@ function InitLocalOSDependingSettings {
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
@ -2320,7 +2331,7 @@ function WildcardFileExists () {
fi fi
} }
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap.sh which will replace includes with actual code
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@ -2678,7 +2689,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -2700,7 +2711,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -2798,9 +2809,7 @@ function CheckReplicas {
_CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & _CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" $retval Logger "Cancelling task." "CRITICAL" $retval
@ -3018,7 +3027,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -3040,7 +3049,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -3178,7 +3187,6 @@ function HandleLocks {
pids="$pids;$targetPid" pids="$pids;$targetPid"
fi fi
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
@ -3284,7 +3292,6 @@ function UnlockReplicas {
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
fi fi
} }
@ -3575,13 +3582,12 @@ function conflictList {
return $retval return $retval
fi fi
join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.comapre.$SCRIPT_PID.$TSTAMP" join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP"
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Cannot create conflict list file." "ERROR" Logger "Cannot create conflict list file." "ERROR"
return $retval return $retval
fi fi
fi fi
} }
@ -3617,9 +3623,7 @@ function syncAttrs {
fi fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$? retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3653,9 +3657,7 @@ function syncAttrs {
_getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" & _getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $pids
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Getting ctime attributes failed." "CRITICAL" $retval Logger "Getting ctime attributes failed." "CRITICAL" $retval
@ -3708,9 +3710,7 @@ function syncAttrs {
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$? retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -4003,7 +4003,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -4025,7 +4025,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -4206,27 +4206,51 @@ function Initialize {
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
pid=${pid%:*} pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then if [ "$pid" == "$initiatorPid" ]; then
Logger "Failed to create initialization files for initiator." "ERROR" Logger "Failed to create initialization treeList files for initiator." "ERROR"
elif [ "$pid" == "$targetPid" ]; then elif [ "$pid" == "$targetPid" ]; then
Logger "Failed to create initialization files for target." "ERROR" Logger "Failed to create initialization treeList files for target." "ERROR"
fi fi
done done
exit 1 exit 1
resumeTarget="${SYNC_ACTION[8]}" resumeTarget="${SYNC_ACTION[8]}"
fi fi
timestampList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__type]}${INITIATOR[$__treeAfterFile]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__type]}${INITIATOR[$__timestampAfterFile]}" &
initiatorPid="$!"
timestampList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${TARGET[$__type]}${TARGET[$__treeAfterFile]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${TARGET[$__type]}${INITIATOR[$__timestampAfterFile]}" &
targetPid="$!"
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then
Logger "Failed to create initialization timestamp files for initiator." "ERROR"
elif [ "$pid" == "$targetPid" ]; then
Logger "Failed to create initialization timestamp files for target." "ERROR"
fi
done
exit 1
resumeTarget="${SYNC_ACTION[8]}"
fi
} }
###### Sync function in 9 steps ###### Sync function in 9 steps
###### ######
###### Step 0a & 0b: Create current file list of replicas ###### Step 0a & 0b: Create current file list of replicas
###### Step 1a & 1b: Create deleted file list of replicas ###### Step 1a & 1b: Create deleted file list of replicas
###### Step 2a & 2b: Create ctime & mtime file list of replicas ###### Step 2a & 2b: Create current ctime & mtime file list of replicas
###### Step 3a & 3b: Merge conflict file list ###### Step 3a & 3b: Merge conflict file list
###### Step 4: Update file attributes ###### Step 4: Update file attributes
###### Step 5a & 5b: Update replicas ###### Step 5a & 5b: Update replicas
###### Step 6a & 6b: Propagate deletions on replicas ###### Step 6a & 6b: Propagate deletions on replicas
###### Step 7a & 7b: Create after run file list of replicas ###### Step 7a & 7b: Create after run file list of replicas
###### Step 8a & 8b: Create ctime & mtime file lost of replicas ###### Step 8a & 8b: Create after run ctime & mtime file list of replicas
function Sync { function Sync {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -4304,9 +4328,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4351,9 +4373,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4402,9 +4422,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4447,9 +4465,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
if [ "$LOG_CONFLICTS" == "yes" ]; then if [ "$LOG_CONFLICTS" == "yes" ]; then
conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFileNoSuffix]}" "${INITIATOR[$__conflictListFile]}" & conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFileNoSuffix]}" "${INITIATOR[$__conflictListFile]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -4473,9 +4489,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" & syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -4500,9 +4514,7 @@ function Sync {
if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}" resumeTarget="${SYNC_ACTION[5]}"
@ -4514,9 +4526,7 @@ function Sync {
fi fi
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}" resumeInitiator="${SYNC_ACTION[5]}"
@ -4529,9 +4539,7 @@ function Sync {
else else
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}" resumeInitiator="${SYNC_ACTION[5]}"
@ -4543,9 +4551,7 @@ function Sync {
fi fi
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}" resumeTarget="${SYNC_ACTION[5]}"
@ -4570,9 +4576,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4618,9 +4622,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4669,9 +4671,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4925,7 +4925,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -4947,7 +4947,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -5032,9 +5032,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1 exit 1
fi fi
@ -5052,9 +5050,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1 exit 1
fi fi
@ -5125,28 +5121,27 @@ function LogConflicts {
local body local body
# We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX # We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX
if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS"
fi fi
( (
_LOGGER_PREFIX="" _LOGGER_PREFIX=""
if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
while read -r line; do while read -r line; do
echo "${INITIATOR[$__replicaDir]}$(echo $line | awk -F';' '{print $1}') -- ${TARGET[$__replicaDir]}$(echo $line | awk -F';' '{print $1}')" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" echo "${INITIATOR[$__replicaDir]}$(echo $line | awk -F';' '{print $1}') << >> ${TARGET[$__replicaDir]}$(echo $line | awk -F';' '{print $1}')" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
done < "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" done < "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP"
Logger "$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})" "ALWAYS" Logger "$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})" "ALWAYS"
if [ "$ALERT_CONFLICTS" == "yes" ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
fi fi
) )
if [ "$ALERT_CONFLICTS" == "yes" ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
} }
function Init { function Init {
@ -5446,12 +5441,14 @@ function SyncOnChanges {
Logger "#### Monitoring now." "NOTICE" Logger "#### Monitoring now." "NOTICE"
if [ "$LOCAL_OS" == "MacOSX" ]; then if [ "$LOCAL_OS" == "MacOSX" ]; then
fswatch $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null & fswatch $RSYNC_PATTERNS --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout # Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
ExecTasks $! "MonitorMacOSXWait" false 0 0 0 $MAX_WAIT true 1 0 ExecTasks $! "MonitorMacOSXWait" false 0 0 0 $MAX_WAIT true 1 0
#ExecTasks "MonitorMacOSXWait" 0 0 0 $MAX_WAIT 1 0 true false true false $! elif [ "$LOCAL_OS" "BSD" ]; then
# BSD version of inotifywait does not support multiple --exclude statements
inotifywait --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
else else
inotifywait $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & inotifywait $RSYNC_PATTERNS --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
wait $! wait $!
fi fi
retval=$? retval=$?
@ -5601,7 +5598,7 @@ function GetCommandlineArguments {
;; ;;
--initialize) --initialize)
INITIALIZE="yes" INITIALIZE="yes"
opts=$opts "--initialize" opts=$opts" --initialize"
;; ;;
--no-prefix) --no-prefix)
opts=$opts" --no-prefix" opts=$opts" --no-prefix"
@ -5675,9 +5672,13 @@ if [ $_QUICK_SYNC -eq 2 ]; then
if [ $(IsInteger $MIN_WAIT) -ne 1 ]; then if [ $(IsInteger $MIN_WAIT) -ne 1 ]; then
MIN_WAIT=30 MIN_WAIT=30
fi fi
else # First character shouldn't be '-' when config file given
elif [ "${1:0:1}" != "-" ]; then
ConfigFile="${1}" ConfigFile="${1}"
LoadConfigFile "$ConfigFile" LoadConfigFile "$ConfigFile"
else
Logger "Wrong arguments given. Expecting a config file or initiator and target arguments." "CRITICAL"
exit 1
fi fi
if [ "$LOGFILE" == "" ]; then if [ "$LOGFILE" == "" ]; then

166
dev/debug_osync_target_helper.sh
View File

@ -7,16 +7,10 @@ PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017061901 PROGRAM_BUILD=2017061901
IS_STABLE=no IS_STABLE=no
_OFUNCTIONS_VERSION=2.3.0-beta1
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" _OFUNCTIONS_BUILD=2018080801
#command line arguments don't take -AaqV for example
_OFUNCTIONS_VERSION=2.3.0-dev
_OFUNCTIONS_BUILD=2018062504
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
## PROGRAM=program-name ## PROGRAM=program-name
## INSTANCE_ID=program-instance-name ## INSTANCE_ID=program-instance-name
@ -26,8 +20,6 @@ _OFUNCTIONS_BOOTSTRAP=true
## _LOGGER_ERR_ONLY=true/false ## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/"" ## _LOGGER_PREFIX="date"/"time"/""
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP ## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
@ -215,7 +207,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -237,7 +229,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -352,7 +344,6 @@ function KillChilds {
fi fi
if kill -0 "$pid" > /dev/null 2>&1; then if kill -0 "$pid" > /dev/null 2>&1; then
#TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL" Logger "Bogus pgrep implementation." "CRITICAL"
@ -474,6 +465,12 @@ function SendAlert {
# encryption can be set to tls, ssl or none # encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional # smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"
# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc
#set ttycharset=iso-8859-1
#set sendcharsets=iso-8859-1
#set encoding=8bit
function SendEmail { function SendEmail {
local subject="${1}" local subject="${1}"
local message="${2}" local message="${2}"
@ -496,11 +493,16 @@ function SendEmail {
local i local i
for i in "${destinationMails}"; do if [ "${destinationMails[@]}" != "" ]; then
if [ $(CheckRFC822 "$i") -ne 1 ]; then for i in "${destinationMails[@]}"; do
Logger "Given email [$i] does not seem to be valid." "WARN" if [ $(CheckRFC822 "$i") -ne 1 ]; then
fi Logger "Given email [$i] does not seem to be valid." "WARN"
done fi
done
else
Logger "No valid email adresses given." "WARN"
return 1
fi
# Prior to sending an email, convert its body if needed # Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then if [ "$MAIL_BODY_CHARSET" != "" ]; then
@ -772,33 +774,33 @@ function ParallelExec {
function ExecTasks { function ExecTasks {
# Mandatory arguments # Mandatory arguments
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments # Optional arguments
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments # Parallelism specific arguments
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i local i
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -821,40 +823,40 @@ function ExecTasks {
IFS=';' read -r -a validExitCodes <<< "$validExitCodes" IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
# ParallelExec specific variables # ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing pids of commands currently run local commandsArrayPid=() # Array containing pids of commands currently run
local postponedRetryCount=0 # Number of current postponed commands retries local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0 local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ? local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed local needsPostponing # Does currentCommand need to be postponed
local temp local temp
# Common variables # Common variables
local pid # Current pid working on local pid # Current pid working on
local pidState # State of the process local pidState # State of the process
local mainItemCount=0 # number of given items (pids or commands) local mainItemCount=0 # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true) local readFromFile # Should we read pids / commands from a file (true)
local counter=0 local counter=0
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
local functionMode local functionMode
@ -907,7 +909,7 @@ function ExecTasks {
counter=$mainItemCount counter=$mainItemCount
fi fi
Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
function _ExecTasksTimeCheck { function _ExecTasksTimeCheck {
@ -1019,11 +1021,11 @@ function ExecTasks {
else else
# pid is dead, get its exit code from wait command # pid is dead, get its exit code from wait command
wait $pid wait $pid
retval=$? #TODO: do we use retval codes somehow ?? where retval=$?
# Check for valid exit codes # Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
if [ "$functionMode" == "ParallelExec" ]; then if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi fi
@ -1039,22 +1041,22 @@ function ExecTasks {
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
fi fi
fi fi
hasPids=true ##__WITH_PARANOIA_DEBUG hasPids=true ##__WITH_PARANOIA_DEBUG
fi fi
done done
# hasPids can be false on last iteration in ParallelExec mode # hasPids can be false on last iteration in ParallelExec mode
if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $sleepTime sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG _PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG fi ##__WITH_PARANOIA_DEBUG
} }
@ -1243,6 +1245,13 @@ function EscapeSpaces {
echo "${string// /\\ }" echo "${string// /\\ }"
} }
# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")"
function EscapeDoubleQuotes {
local value="${1}"
echo "${value//\"/\\\"}"
}
function IsNumericExpand { function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
@ -1331,6 +1340,8 @@ function CheckRFC822 {
function UrlEncode { function UrlEncode {
local length="${#1}" local length="${#1}"
local i
local LANG=C local LANG=C
for i in $(seq 0 $((length-1))); do for i in $(seq 0 $((length-1))); do
local c="${1:i:1}" local c="${1:i:1}"
@ -1607,6 +1618,9 @@ GetOs
ENDSSH ENDSSH
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL" Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
fi
exit 1 exit 1
fi fi
@ -2025,7 +2039,7 @@ function InitLocalOSDependingSettings {
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
@ -2479,7 +2493,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -2501,7 +2515,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then

15
install.sh
View File

@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh" SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2018062601 SCRIPT_BUILD=2018070201
## osync / obackup / pmocr / zsnap install script ## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@ -140,7 +140,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -162,7 +162,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -269,6 +269,8 @@ function Logger {
function UrlEncode { function UrlEncode {
local length="${#1}" local length="${#1}"
local i
local LANG=C local LANG=C
for i in $(seq 0 $((length-1))); do for i in $(seq 0 $((length-1))); do
local c="${1:i:1}" local c="${1:i:1}"
@ -448,7 +450,7 @@ function GetInit {
Logger "Detected initV." "SIMPLE" Logger "Detected initV." "SIMPLE"
fi fi
else else
Logger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron." "SIMPLE" Logger "Can't detect initV, systemd or openRC. Service files won't be installed. You can still run $PROGRAM manually or via cron." "SIMPLE"
init="none" init="none"
fi fi
} }
@ -570,7 +572,6 @@ function CopyServiceFiles {
CreateDir "$SERVICE_DIR_SYSTEMD_USER" CreateDir "$SERVICE_DIR_SYSTEMD_USER"
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
fi fi
Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." "SIMPLE" Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." "SIMPLE"
Logger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." "SIMPLE" Logger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." "SIMPLE"
Logger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." "SIMPLE" Logger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." "SIMPLE"
@ -582,14 +583,14 @@ function CopyServiceFiles {
Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." "SIMPLE" Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." "SIMPLE"
Logger "Can be activated with [service $SERVICE_FILE_INIT start]." "SIMPLE" Logger "Can be activated with [service $SERVICE_FILE_INIT start]." "SIMPLE"
Logger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]." "SIMPLE" Logger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]." "SIMPLE"
elif ([ "$init" == "openrc" && [ -f "$SCRIPT_PATH/$SERVICE_FILE_OPENRC" ] && [ -d "$SERVICE_DIR_OPENRC" ]); then elif ([ "$init" == "openrc" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_OPENRC" ] && [ -d "$SERVICE_DIR_OPENRC" ]); then
# Rename service to usual service file # Rename service to usual service file
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_OPENRC" "$SERVICE_FILE_OPENRC" "$SERVICE_FILE_INIT" "755" "" "" true CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_OPENRC" "$SERVICE_FILE_OPENRC" "$SERVICE_FILE_INIT" "755" "" "" true
Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_OPENRC]." "SIMPLE" Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_OPENRC]." "SIMPLE"
Logger "Can be activated with [rc-update add $SERVICE_NAME.instance] where instance is a configuration file found in /etc/osync." "SIMPLE" Logger "Can be activated with [rc-update add $SERVICE_NAME.instance] where instance is a configuration file found in /etc/osync." "SIMPLE"
else else
Logger "Cannot define what init style is in use on this system. Skipping service file installation." "SIMPLE" Logger "Cannot properly find how to deal with init on this system. Skipping service file installation." "SIMPLE"
fi fi
} }

277
osync.sh
View File

@ -5,26 +5,17 @@
#Check dryruns with nosuffix mode for timestampList #Check dryruns with nosuffix mode for timestampList
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong" AUTHOR="(C) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.5-dev PROGRAM_VERSION=1.3.0-beta1
PROGRAM_BUILD=2018062506 PROGRAM_BUILD=2018080801
IS_STABLE=no IS_STABLE=no
#TODO: tidy up ExecTasks comments
_OFUNCTIONS_VERSION=2.3.0-beta1
_OFUNCTIONS_BUILD=2018080801
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling"
#command line arguments don't take -AaqV for example
_OFUNCTIONS_VERSION=2.3.0-dev
_OFUNCTIONS_BUILD=2018062504
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
## PROGRAM=program-name ## PROGRAM=program-name
## INSTANCE_ID=program-instance-name ## INSTANCE_ID=program-instance-name
@ -34,8 +25,6 @@ _OFUNCTIONS_BOOTSTRAP=true
## _LOGGER_ERR_ONLY=true/false ## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/"" ## _LOGGER_PREFIX="date"/"time"/""
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP ## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
@ -218,7 +207,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -240,7 +229,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -345,7 +334,6 @@ function KillChilds {
fi fi
if kill -0 "$pid" > /dev/null 2>&1; then if kill -0 "$pid" > /dev/null 2>&1; then
#TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL" Logger "Bogus pgrep implementation." "CRITICAL"
@ -464,6 +452,12 @@ function SendAlert {
# encryption can be set to tls, ssl or none # encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional # smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"
# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc
#set ttycharset=iso-8859-1
#set sendcharsets=iso-8859-1
#set encoding=8bit
function SendEmail { function SendEmail {
local subject="${1}" local subject="${1}"
local message="${2}" local message="${2}"
@ -485,11 +479,16 @@ function SendEmail {
local i local i
for i in "${destinationMails}"; do if [ "${destinationMails[@]}" != "" ]; then
if [ $(CheckRFC822 "$i") -ne 1 ]; then for i in "${destinationMails[@]}"; do
Logger "Given email [$i] does not seem to be valid." "WARN" if [ $(CheckRFC822 "$i") -ne 1 ]; then
fi Logger "Given email [$i] does not seem to be valid." "WARN"
done fi
done
else
Logger "No valid email adresses given." "WARN"
return 1
fi
# Prior to sending an email, convert its body if needed # Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then if [ "$MAIL_BODY_CHARSET" != "" ]; then
@ -744,28 +743,28 @@ function ParallelExec {
function ExecTasks { function ExecTasks {
# Mandatory arguments # Mandatory arguments
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments # Optional arguments
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments # Parallelism specific arguments
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i local i
@ -791,38 +790,38 @@ function ExecTasks {
IFS=';' read -r -a validExitCodes <<< "$validExitCodes" IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
# ParallelExec specific variables # ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing pids of commands currently run local commandsArrayPid=() # Array containing pids of commands currently run
local postponedRetryCount=0 # Number of current postponed commands retries local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0 local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ? local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed local needsPostponing # Does currentCommand need to be postponed
local temp local temp
# Common variables # Common variables
local pid # Current pid working on local pid # Current pid working on
local pidState # State of the process local pidState # State of the process
local mainItemCount=0 # number of given items (pids or commands) local mainItemCount=0 # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true) local readFromFile # Should we read pids / commands from a file (true)
local counter=0 local counter=0
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition local executeCommand # Boolean to check if currentCommand can be executed given a condition
local functionMode local functionMode
@ -987,11 +986,11 @@ function ExecTasks {
else else
# pid is dead, get its exit code from wait command # pid is dead, get its exit code from wait command
wait $pid wait $pid
retval=$? #TODO: do we use retval codes somehow ?? where retval=$?
# Check for valid exit codes # Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
if [ "$functionMode" == "ParallelExec" ]; then if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi fi
@ -1202,6 +1201,13 @@ function EscapeSpaces {
echo "${string// /\\ }" echo "${string// /\\ }"
} }
# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")"
function EscapeDoubleQuotes {
local value="${1}"
echo "${value//\"/\\\"}"
}
function IsNumericExpand { function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
@ -1290,6 +1296,8 @@ function CheckRFC822 {
function UrlEncode { function UrlEncode {
local length="${#1}" local length="${#1}"
local i
local LANG=C local LANG=C
for i in $(seq 0 $((length-1))); do for i in $(seq 0 $((length-1))); do
local c="${1:i:1}" local c="${1:i:1}"
@ -1510,6 +1518,9 @@ GetOs
ENDSSH ENDSSH
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL" Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
fi
exit 1 exit 1
fi fi
@ -1912,7 +1923,7 @@ function InitLocalOSDependingSettings {
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
@ -2169,7 +2180,7 @@ function WildcardFileExists () {
fi fi
} }
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap.sh which will replace includes with actual code
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@ -2518,7 +2529,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -2540,7 +2551,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -2632,9 +2643,7 @@ function CheckReplicas {
_CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" & _CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" $retval Logger "Cancelling task." "CRITICAL" $retval
@ -2846,7 +2855,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -2868,7 +2877,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -3000,7 +3009,6 @@ function HandleLocks {
pids="$pids;$targetPid" pids="$pids;$targetPid"
fi fi
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
@ -3103,7 +3111,6 @@ function UnlockReplicas {
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
fi fi
} }
@ -3388,13 +3395,12 @@ function conflictList {
return $retval return $retval
fi fi
join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.comapre.$SCRIPT_PID.$TSTAMP" join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP"
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Cannot create conflict list file." "ERROR" Logger "Cannot create conflict list file." "ERROR"
return $retval return $retval
fi fi
fi fi
} }
@ -3429,9 +3435,7 @@ function syncAttrs {
fi fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$? retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3465,9 +3469,7 @@ function syncAttrs {
_getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" & _getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $pids
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Getting ctime attributes failed." "CRITICAL" $retval Logger "Getting ctime attributes failed." "CRITICAL" $retval
@ -3520,9 +3522,7 @@ function syncAttrs {
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$? retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3808,7 +3808,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -3830,7 +3830,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -4004,27 +4004,51 @@ function Initialize {
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
pid=${pid%:*} pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then if [ "$pid" == "$initiatorPid" ]; then
Logger "Failed to create initialization files for initiator." "ERROR" Logger "Failed to create initialization treeList files for initiator." "ERROR"
elif [ "$pid" == "$targetPid" ]; then elif [ "$pid" == "$targetPid" ]; then
Logger "Failed to create initialization files for target." "ERROR" Logger "Failed to create initialization treeList files for target." "ERROR"
fi fi
done done
exit 1 exit 1
resumeTarget="${SYNC_ACTION[8]}" resumeTarget="${SYNC_ACTION[8]}"
fi fi
timestampList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__type]}${INITIATOR[$__treeAfterFile]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__type]}${INITIATOR[$__timestampAfterFile]}" &
initiatorPid="$!"
timestampList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${TARGET[$__type]}${TARGET[$__treeAfterFile]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${TARGET[$__type]}${INITIATOR[$__timestampAfterFile]}" &
targetPid="$!"
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ "$pid" == "$initiatorPid" ]; then
Logger "Failed to create initialization timestamp files for initiator." "ERROR"
elif [ "$pid" == "$targetPid" ]; then
Logger "Failed to create initialization timestamp files for target." "ERROR"
fi
done
exit 1
resumeTarget="${SYNC_ACTION[8]}"
fi
} }
###### Sync function in 9 steps ###### Sync function in 9 steps
###### ######
###### Step 0a & 0b: Create current file list of replicas ###### Step 0a & 0b: Create current file list of replicas
###### Step 1a & 1b: Create deleted file list of replicas ###### Step 1a & 1b: Create deleted file list of replicas
###### Step 2a & 2b: Create ctime & mtime file list of replicas ###### Step 2a & 2b: Create current ctime & mtime file list of replicas
###### Step 3a & 3b: Merge conflict file list ###### Step 3a & 3b: Merge conflict file list
###### Step 4: Update file attributes ###### Step 4: Update file attributes
###### Step 5a & 5b: Update replicas ###### Step 5a & 5b: Update replicas
###### Step 6a & 6b: Propagate deletions on replicas ###### Step 6a & 6b: Propagate deletions on replicas
###### Step 7a & 7b: Create after run file list of replicas ###### Step 7a & 7b: Create after run file list of replicas
###### Step 8a & 8b: Create ctime & mtime file lost of replicas ###### Step 8a & 8b: Create after run ctime & mtime file list of replicas
function Sync { function Sync {
@ -4101,9 +4125,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4148,9 +4170,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4199,9 +4219,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4244,9 +4262,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
if [ "$LOG_CONFLICTS" == "yes" ]; then if [ "$LOG_CONFLICTS" == "yes" ]; then
conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFileNoSuffix]}" "${INITIATOR[$__conflictListFile]}" & conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFileNoSuffix]}" "${INITIATOR[$__conflictListFile]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -4270,9 +4286,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" & syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -4297,9 +4311,7 @@ function Sync {
if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}" resumeTarget="${SYNC_ACTION[5]}"
@ -4311,9 +4323,7 @@ function Sync {
fi fi
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}" resumeInitiator="${SYNC_ACTION[5]}"
@ -4326,9 +4336,7 @@ function Sync {
else else
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" & syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}" resumeInitiator="${SYNC_ACTION[5]}"
@ -4340,9 +4348,7 @@ function Sync {
fi fi
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" & syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}" resumeTarget="${SYNC_ACTION[5]}"
@ -4367,9 +4373,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4415,9 +4419,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4466,9 +4468,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false initiatorFail=false
@ -4716,7 +4716,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -4738,7 +4738,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -4817,9 +4817,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1 exit 1
fi fi
@ -4837,9 +4835,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!" pids="$pids;$!"
fi fi
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING ExecTasks $pids "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1 exit 1
fi fi
@ -4906,28 +4902,27 @@ function LogConflicts {
local body local body
# We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX # We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX
if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS"
fi fi
( (
_LOGGER_PREFIX="" _LOGGER_PREFIX=""
if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
while read -r line; do while read -r line; do
echo "${INITIATOR[$__replicaDir]}$(echo $line | awk -F';' '{print $1}') -- ${TARGET[$__replicaDir]}$(echo $line | awk -F';' '{print $1}')" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" echo "${INITIATOR[$__replicaDir]}$(echo $line | awk -F';' '{print $1}') << >> ${TARGET[$__replicaDir]}$(echo $line | awk -F';' '{print $1}')" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
done < "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" done < "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP"
Logger "$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})" "ALWAYS" Logger "$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})" "ALWAYS"
if [ "$ALERT_CONFLICTS" == "yes" ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
fi fi
) )
if [ "$ALERT_CONFLICTS" == "yes" ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat ${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]})"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
} }
function Init { function Init {
@ -5223,12 +5218,14 @@ function SyncOnChanges {
Logger "#### Monitoring now." "NOTICE" Logger "#### Monitoring now." "NOTICE"
if [ "$LOCAL_OS" == "MacOSX" ]; then if [ "$LOCAL_OS" == "MacOSX" ]; then
fswatch $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null & fswatch $RSYNC_PATTERNS --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout # Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
ExecTasks $! "MonitorMacOSXWait" false 0 0 0 $MAX_WAIT true 1 0 ExecTasks $! "MonitorMacOSXWait" false 0 0 0 $MAX_WAIT true 1 0
#ExecTasks "MonitorMacOSXWait" 0 0 0 $MAX_WAIT 1 0 true false true false $! elif [ "$LOCAL_OS" "BSD" ]; then
# BSD version of inotifywait does not support multiple --exclude statements
inotifywait --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
else else
inotifywait $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & inotifywait $RSYNC_PATTERNS --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
wait $! wait $!
fi fi
retval=$? retval=$?
@ -5378,7 +5375,7 @@ function GetCommandlineArguments {
;; ;;
--initialize) --initialize)
INITIALIZE="yes" INITIALIZE="yes"
opts=$opts "--initialize" opts=$opts" --initialize"
;; ;;
--no-prefix) --no-prefix)
opts=$opts" --no-prefix" opts=$opts" --no-prefix"
@ -5452,9 +5449,13 @@ if [ $_QUICK_SYNC -eq 2 ]; then
if [ $(IsInteger $MIN_WAIT) -ne 1 ]; then if [ $(IsInteger $MIN_WAIT) -ne 1 ]; then
MIN_WAIT=30 MIN_WAIT=30
fi fi
else # First character shouldn't be '-' when config file given
elif [ "${1:0:1}" != "-" ]; then
ConfigFile="${1}" ConfigFile="${1}"
LoadConfigFile "$ConfigFile" LoadConfigFile "$ConfigFile"
else
Logger "Wrong arguments given. Expecting a config file or initiator and target arguments." "CRITICAL"
exit 1
fi fi
if [ "$LOGFILE" == "" ]; then if [ "$LOGFILE" == "" ]; then

144
osync_target_helper.sh
View File

@ -7,16 +7,10 @@ PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017061901 PROGRAM_BUILD=2017061901
IS_STABLE=no IS_STABLE=no
_OFUNCTIONS_VERSION=2.3.0-beta1
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" _OFUNCTIONS_BUILD=2018080801
#command line arguments don't take -AaqV for example
_OFUNCTIONS_VERSION=2.3.0-dev
_OFUNCTIONS_BUILD=2018062504
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
## PROGRAM=program-name ## PROGRAM=program-name
## INSTANCE_ID=program-instance-name ## INSTANCE_ID=program-instance-name
@ -26,8 +20,6 @@ _OFUNCTIONS_BOOTSTRAP=true
## _LOGGER_ERR_ONLY=true/false ## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/"" ## _LOGGER_PREFIX="date"/"time"/""
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP ## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
@ -210,7 +202,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -232,7 +224,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
@ -337,7 +329,6 @@ function KillChilds {
fi fi
if kill -0 "$pid" > /dev/null 2>&1; then if kill -0 "$pid" > /dev/null 2>&1; then
#TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL" Logger "Bogus pgrep implementation." "CRITICAL"
@ -456,6 +447,12 @@ function SendAlert {
# encryption can be set to tls, ssl or none # encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional # smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"
# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc
#set ttycharset=iso-8859-1
#set sendcharsets=iso-8859-1
#set encoding=8bit
function SendEmail { function SendEmail {
local subject="${1}" local subject="${1}"
local message="${2}" local message="${2}"
@ -477,11 +474,16 @@ function SendEmail {
local i local i
for i in "${destinationMails}"; do if [ "${destinationMails[@]}" != "" ]; then
if [ $(CheckRFC822 "$i") -ne 1 ]; then for i in "${destinationMails[@]}"; do
Logger "Given email [$i] does not seem to be valid." "WARN" if [ $(CheckRFC822 "$i") -ne 1 ]; then
fi Logger "Given email [$i] does not seem to be valid." "WARN"
done fi
done
else
Logger "No valid email adresses given." "WARN"
return 1
fi
# Prior to sending an email, convert its body if needed # Prior to sending an email, convert its body if needed
if [ "$MAIL_BODY_CHARSET" != "" ]; then if [ "$MAIL_BODY_CHARSET" != "" ]; then
@ -736,28 +738,28 @@ function ParallelExec {
function ExecTasks { function ExecTasks {
# Mandatory arguments # Mandatory arguments
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments # Optional arguments
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments # Parallelism specific arguments
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i local i
@ -783,38 +785,38 @@ function ExecTasks {
IFS=';' read -r -a validExitCodes <<< "$validExitCodes" IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
# ParallelExec specific variables # ParallelExec specific variables
local auxItemCount=0 # Number of conditional commands local auxItemCount=0 # Number of conditional commands
local commandsArray=() # Array containing commands local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing pids of commands currently run local commandsArrayPid=() # Array containing pids of commands currently run
local postponedRetryCount=0 # Number of current postponed commands retries local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0 local postponedCounter=0
local isPostponedCommand=false # Is the current command from a postponed file ? local isPostponedCommand=false # Is the current command from a postponed file ?
local postponedExecTime=0 # How much time has passed since last postponed condition was checked local postponedExecTime=0 # How much time has passed since last postponed condition was checked
local needsPostponing # Does currentCommand need to be postponed local needsPostponing # Does currentCommand need to be postponed
local temp local temp
# Common variables # Common variables
local pid # Current pid working on local pid # Current pid working on
local pidState # State of the process local pidState # State of the process
local mainItemCount=0 # number of given items (pids or commands) local mainItemCount=0 # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true) local readFromFile # Should we read pids / commands from a file (true)
local counter=0 local counter=0
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process local retval=0 # return value of monitored pid process
local subRetval=0 # return value of condition commands local subRetval=0 # return value of condition commands
local errorcount=0 # Number of pids that finished with errors local errorcount=0 # Number of pids that finished with errors
local pidsArray # Array of currently running pids local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids for next iteration local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition local executeCommand # Boolean to check if currentCommand can be executed given a condition
local functionMode local functionMode
@ -979,11 +981,11 @@ function ExecTasks {
else else
# pid is dead, get its exit code from wait command # pid is dead, get its exit code from wait command
wait $pid wait $pid
retval=$? #TODO: do we use retval codes somehow ?? where retval=$?
# Check for valid exit codes # Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
if [ "$functionMode" == "ParallelExec" ]; then if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi fi
@ -1194,6 +1196,13 @@ function EscapeSpaces {
echo "${string// /\\ }" echo "${string// /\\ }"
} }
# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")"
function EscapeDoubleQuotes {
local value="${1}"
echo "${value//\"/\\\"}"
}
function IsNumericExpand { function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
@ -1282,6 +1291,8 @@ function CheckRFC822 {
function UrlEncode { function UrlEncode {
local length="${#1}" local length="${#1}"
local i
local LANG=C local LANG=C
for i in $(seq 0 $((length-1))); do for i in $(seq 0 $((length-1))); do
local c="${1:i:1}" local c="${1:i:1}"
@ -1502,6 +1513,9 @@ GetOs
ENDSSH ENDSSH
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL" Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
fi
exit 1 exit 1
fi fi
@ -1904,7 +1918,7 @@ function InitLocalOSDependingSettings {
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
@ -2349,7 +2363,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[91m$value\e[0m" true _Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi fi
@ -2371,7 +2385,7 @@ function RemoteLogger {
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value" _Logger "" "$prefix$value"
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then