From 42ed3b156ad52b8c9a85ed3e1734dcf0d3146bfc Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 26 Jun 2018 00:23:01 +0200 Subject: [PATCH] Rebuilt targets --- dev/debug_osync.sh | 377 ++++++++++++++++++++++-------- dev/debug_osync_target_helper.sh | 205 +++++++++++----- install.sh | 343 +++++++++++++++++++++------ osync.sh | 385 +++++++++++++++++++++++-------- osync_target_helper.sh | 205 +++++++++++----- 5 files changed, 1130 insertions(+), 385 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 1afcb7e..89fa49f 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -8,9 +8,11 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2.5-dev -PROGRAM_BUILD=2018032201 +PROGRAM_BUILD=2018062506 IS_STABLE=no +#TODO: tidy up ExecTasks comments + ##### Execution order #__WITH_PARANOIA_DEBUG ##### Function Name Is parallel #__WITH_PARANOIA_DEBUG @@ -44,14 +46,10 @@ IS_STABLE=no #TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" -#done: add checkRFC function (and use it for --destination-mails) -#done: ExecTasks still needs some better call argument list -#done: ExecTasks sub function relocate -#done: SendMail and SendEmail convert functions inverted, check on osync and obackup #command line arguments don't take -AaqV for example _OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018031501 +_OFUNCTIONS_BUILD=2018062504 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -118,9 +116,6 @@ fi SCRIPT_PID=$$ -# TODO: Check if %N works on MacOS -TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') - LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) @@ -148,6 +143,46 @@ else RUN_DIR=. fi +#### PoorMansRandomGenerator SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + + local minimum=1 + local maximum + local n=0 + + if [ "$digits" == "" ]; then + digits=5 + fi + + # Minimum already has a digit + for n in $(seq 1 $((digits-1))); do + minimum=$minimum"0" + maximum=$maximum"9" + done + maximum=$maximum"9" + + #n=0; while [ $n -lt $minimum ]; do n=$n$(dd if=/dev/urandom bs=100 count=1 2>/dev/null | tr -cd '0-9'); done; n=$(echo $n | sed -e 's/^0//') + # bs=19 since if real random strikes, having a 19 digits number is not supported + while [ $n -lt $minimum ] || [ $n -gt $maximum ]; do + if [ $n -lt $minimum ]; then + # Add numbers + n=$n$(dd if=/dev/urandom bs=19 count=1 2>/dev/null | tr -cd '0-9') + n=$(echo $n | sed -e 's/^0//') + if [ "$n" == "" ]; then + n=0 + fi + elif [ $n -gt $maximum ]; then + n=$(echo $n | sed 's/.$//') + fi + done + echo $n +} +#### PoorMansRandomGenerator SUBSET END #### + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) # Default alert attachment filename ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" @@ -163,7 +198,6 @@ function Dummy { sleep $SLEEP_TIME } -#### Logger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -179,8 +213,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -270,6 +307,7 @@ function RemoteLogger { # VERBOSE sent to stdout if _LOGGER_VERBOSE = true # ALWAYS is sent to stdout unless _LOGGER_SILENT = true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level @@ -326,35 +364,18 @@ function Logger { _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } -#### Logger SUBSET END #### - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { @@ -368,7 +389,7 @@ function KillChilds { fi if kill -0 "$pid" > /dev/null 2>&1; then - # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment + #TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" @@ -1035,11 +1056,11 @@ function ExecTasks { else # pid is dead, get its exit code from wait command wait $pid - retval=$? + retval=$? #TODO: do we use retval codes somehow ?? where # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then - Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi @@ -1280,25 +1301,23 @@ function IsNumeric { fi } -# Checks email address validity -function CheckRFC822 { - local mail="${1}" - local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - - if [[ $mail =~ $rfc822 ]]; then - echo 1 - else - echo 0 - fi -} - +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -1333,12 +1352,24 @@ function HumanToNumeric { echo $value } -## from https://gist.github.com/cdown/1163649 +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} + +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -1441,10 +1472,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -1524,6 +1579,8 @@ function GetOs { local localOsVar local localOsName local localOsVer + local localOsBitness + local localOsFamily local osInfo="/etc/os-release" @@ -1550,9 +1607,36 @@ function GetOs { localOsName="${localOsName##*=}" localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - echo "$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" } GetOs @@ -2491,13 +2575,23 @@ function TrapError { (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } # Converts human readable sizes into integer kilobyte sizes @@ -2545,8 +2639,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -2848,13 +2945,23 @@ function ArrayContains () { echo 0 return } +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -2872,8 +2979,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3854,8 +3964,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -4074,6 +4187,35 @@ function deletionPropagation { fi } +function Initialize { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG + + Logger "Initializing initiator and target file lists." "NOTICE" + + treeList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__type]}${INITIATOR[$__treeAfterFile]}" & + initiatorPid="$!" + + treeList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${TARGET[$__type]}${INITIATOR[$__treeAfterFile]}" & + targetPid="$!" + + ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING + if [ $? -ne 0 ]; then + IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" + initiatorFail=false + targetFail=false + for pid in "${pidArray[@]}"; do + pid=${pid%:*} + if [ "$pid" == "$initiatorPid" ]; then + Logger "Failed to create initialization files for initiator." "ERROR" + elif [ "$pid" == "$targetPid" ]; then + Logger "Failed to create initialization files for target." "ERROR" + fi + done + exit 1 + resumeTarget="${SYNC_ACTION[8]}" + fi +} + ###### Sync function in 9 steps ###### ###### Step 0a & 0b: Create current file list of replicas @@ -4680,13 +4822,23 @@ function TrapError { (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } # Converts human readable sizes into integer kilobyte sizes @@ -4734,8 +4886,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -4906,7 +5061,7 @@ function SoftDelete { fi } -function _SummaryFromFile { +function _SummaryFromRsyncFile { local replicaPath="${1}" local summaryFile="${2}" local direction="${3}" @@ -4924,6 +5079,20 @@ function _SummaryFromFile { fi } +function _SummaryFromDeleteFile { + local replicaPath="${1}" + local summaryFile="${2}" + local direction="${3}" + + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG + + if [ -f "$summaryFile" ]; then + while read -r file; do + Logger "$direction $replicaPath$file" "ALWAYS" + done < "$summaryFile" + fi +} + function Summary { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -4932,20 +5101,20 @@ function Summary { Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS" - _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID.$TSTAMP" "~ >>" - _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID.$TSTAMP" "~ <<" + _SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID.$TSTAMP" "~ >>" + _SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID.$TSTAMP" "~ <<" - Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS" - _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID.$TSTAMP" "+ >>" - _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID.$TSTAMP" "+ <<" + Logger "File transfers: INITIATOR << >> TARGET (may include file ownership and timestamp attributes)" "ALWAYS" + _SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID.$TSTAMP" "+ >>" + _SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID.$TSTAMP" "+ <<" Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS" if [ "$REMOTE_OPERATION" == "yes" ]; then - _SummaryFromFile "${TARGET[$__replicaDir]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" "- >>" + _SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" "- >>" else - _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID.$TSTAMP" "- >>" + _SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID.$TSTAMP" "- >>" fi - _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID.$TSTAMP" "- <<" + _SummaryFromDeleteFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID.$TSTAMP" "- <<" ) } @@ -4955,9 +5124,13 @@ function LogConflicts { local subject local body + # We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX + if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then + Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" + fi + ( _LOGGER_PREFIX="" - Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" while read -r line; do @@ -5214,6 +5387,7 @@ function Usage { echo "--force-unlock Will override any existing active or dead locks on initiator and target replica" echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead" echo "--no-resume Do not try to resume a failed run. By default, execution is resumed once" + echo "--initialize Create file lists without actually synchronizing anything, this will help setup deletion detections before the first run" echo "" echo "[QUICKSYNC OPTIONS]" @@ -5327,7 +5501,7 @@ sync_on_changes=false _NOLOCKS=false osync_cmd=$0 _SUMMARY=false - +INITIALIZE="no" function GetCommandlineArguments { local isFirstArgument=true @@ -5425,6 +5599,10 @@ function GetCommandlineArguments { LOG_CONFLICTS="yes" opts=$opts" --alert-conflicts" ;; + --initialize) + INITIALIZE="yes" + opts=$opts "--initialize" + ;; --no-prefix) opts=$opts" --no-prefix" _LOGGER_PREFIX="" @@ -5496,7 +5674,6 @@ if [ $_QUICK_SYNC -eq 2 ]; then if [ $(IsInteger $MIN_WAIT) -ne 1 ]; then MIN_WAIT=30 - fi else ConfigFile="${1}" @@ -5550,14 +5727,20 @@ else fi CheckReplicas RunBeforeHook - Main - if [ $? -eq 0 ]; then - SoftDelete - fi - if [ $_SUMMARY == true ]; then - Summary - fi - if [ $LOG_CONFLICTS == "yes" ]; then - LogConflicts + + if [ "$INITIALIZE" == "yes" ]; then + HandleLocks + Initialize + else + Main + if [ $? -eq 0 ]; then + SoftDelete + fi + if [ $_SUMMARY == true ]; then + Summary + fi + if [ $LOG_CONFLICTS == "yes" ]; then + LogConflicts + fi fi fi diff --git a/dev/debug_osync_target_helper.sh b/dev/debug_osync_target_helper.sh index d1e6378..1117c07 100755 --- a/dev/debug_osync_target_helper.sh +++ b/dev/debug_osync_target_helper.sh @@ -9,14 +9,10 @@ IS_STABLE=no #TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" -#done: add checkRFC function (and use it for --destination-mails) -#done: ExecTasks still needs some better call argument list -#done: ExecTasks sub function relocate -#done: SendMail and SendEmail convert functions inverted, check on osync and obackup #command line arguments don't take -AaqV for example _OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018031501 +_OFUNCTIONS_BUILD=2018062504 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -83,9 +79,6 @@ fi SCRIPT_PID=$$ -# TODO: Check if %N works on MacOS -TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') - LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) @@ -113,6 +106,46 @@ else RUN_DIR=. fi +#### PoorMansRandomGenerator SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + + local minimum=1 + local maximum + local n=0 + + if [ "$digits" == "" ]; then + digits=5 + fi + + # Minimum already has a digit + for n in $(seq 1 $((digits-1))); do + minimum=$minimum"0" + maximum=$maximum"9" + done + maximum=$maximum"9" + + #n=0; while [ $n -lt $minimum ]; do n=$n$(dd if=/dev/urandom bs=100 count=1 2>/dev/null | tr -cd '0-9'); done; n=$(echo $n | sed -e 's/^0//') + # bs=19 since if real random strikes, having a 19 digits number is not supported + while [ $n -lt $minimum ] || [ $n -gt $maximum ]; do + if [ $n -lt $minimum ]; then + # Add numbers + n=$n$(dd if=/dev/urandom bs=19 count=1 2>/dev/null | tr -cd '0-9') + n=$(echo $n | sed -e 's/^0//') + if [ "$n" == "" ]; then + n=0 + fi + elif [ $n -gt $maximum ]; then + n=$(echo $n | sed 's/.$//') + fi + done + echo $n +} +#### PoorMansRandomGenerator SUBSET END #### + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) # Default alert attachment filename ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" @@ -128,7 +161,6 @@ function Dummy { sleep $SLEEP_TIME } -#### Logger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -144,8 +176,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -235,6 +270,7 @@ function RemoteLogger { # VERBOSE sent to stdout if _LOGGER_VERBOSE = true # ALWAYS is sent to stdout unless _LOGGER_SILENT = true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level @@ -291,35 +327,18 @@ function Logger { _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } -#### Logger SUBSET END #### - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { @@ -333,7 +352,7 @@ function KillChilds { fi if kill -0 "$pid" > /dev/null 2>&1; then - # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment + #TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" @@ -1000,11 +1019,11 @@ function ExecTasks { else # pid is dead, get its exit code from wait command wait $pid - retval=$? + retval=$? #TODO: do we use retval codes somehow ?? where # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then - Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi @@ -1245,25 +1264,23 @@ function IsNumeric { fi } -# Checks email address validity -function CheckRFC822 { - local mail="${1}" - local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - - if [[ $mail =~ $rfc822 ]]; then - echo 1 - else - echo 0 - fi -} - +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -1298,12 +1315,24 @@ function HumanToNumeric { echo $value } -## from https://gist.github.com/cdown/1163649 +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} + +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -1406,10 +1435,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -1489,6 +1542,8 @@ function GetOs { local localOsVar local localOsName local localOsVer + local localOsBitness + local localOsFamily local osInfo="/etc/os-release" @@ -1515,9 +1570,36 @@ function GetOs { localOsName="${localOsName##*=}" localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - echo "$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" } GetOs @@ -2358,8 +2440,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then diff --git a/install.sh b/install.sh index 94d0f24..508a788 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2017072701 +SCRIPT_BUILD=2018062601 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -45,8 +45,9 @@ function GetCommandlineArguments { Usage ;; *) - Logger "Unknown option '$i'" "CRITICAL" + Logger "Unknown option '$i'" "SIMPLE" Usage + exit ;; esac done @@ -60,6 +61,7 @@ SERVICE_DIR_INIT=$FAKEROOT/etc/init.d # Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user +SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d if [ "$PROGRAM" == "osync" ]; then SERVICE_NAME="osync-srv" @@ -70,6 +72,7 @@ fi SERVICE_FILE_INIT="$SERVICE_NAME" SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service" SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user" +SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc" ## Generic code @@ -82,34 +85,192 @@ else LOG_FILE="./$PROGRAM-install.log" fi -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both +#### RemoteLogger SUBSET #### - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + +# Sub function of Logger +function _Logger { + local logValue="${1}" # Log to file + local stdValue="${2}" # Log to screeen + local toStdErr="${3:-false}" # Log to stderr instead of stdout + + if [ "$logValue" != "" ]; then + echo -e "$logValue" >> "$LOG_FILE" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi + fi + + if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then + if [ $toStdErr == true ]; then + # Force stderr color in subshell + (>&2 echo -e "$stdValue") + + else + echo -e "$stdValue" + fi fi } -# Generic quick logging function -function QuickLogger { - local value="${1}" +# Remote logger similar to below Logger, without log to file and alert flags +function RemoteLogger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level + local retval="${3:-undef}" # optional return value of command - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then + prefix="R $(date) - " else - _QuickLogger "$value" "stdout" + prefix="" + fi + + if [ "$level" == "CRITICAL" ]; then + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true + if [ $_DEBUG == "yes" ]; then + _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true + fi + return + elif [ "$level" == "ERROR" ]; then + _Logger "" "$prefix\e[91m$value\e[0m" true + if [ $_DEBUG == "yes" ]; then + _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true + fi + return + elif [ "$level" == "WARN" ]; then + _Logger "" "$prefix\e[33m$value\e[0m" true + if [ $_DEBUG == "yes" ]; then + _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true + fi + return + elif [ "$level" == "NOTICE" ]; then + if [ $_LOGGER_ERR_ONLY != true ]; then + _Logger "" "$prefix$value" + fi + return + elif [ "$level" == "VERBOSE" ]; then + if [ $_LOGGER_VERBOSE == true ]; then + _Logger "" "$prefix$value" + fi + return + elif [ "$level" == "ALWAYS" ]; then + _Logger "" "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "" "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true fi } -## from https://gist.github.com/cdown/1163649 +#### RemoteLogger SUBSET END #### + +# General log function with log levels: + +# Environment variables +# _LOGGER_SILENT: Disables any output to stdout & stderr +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout + +# Loglevels +# Except for VERBOSE, all loglevels are ALWAYS sent to log file + +# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged +# NOTICE sent to stdout +# VERBOSE sent to stdout if _LOGGER_VERBOSE = true +# ALWAYS is sent to stdout unless _LOGGER_SILENT = true +# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality +function Logger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level + local retval="${3:-undef}" # optional return value of command + + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then + prefix="$(date) - " + else + prefix="" + fi + + ## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup) + value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}" + value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}" + + if [ "$level" == "CRITICAL" ]; then + _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true + ERROR_ALERT=true + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + return + elif [ "$level" == "ERROR" ]; then + _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true + ERROR_ALERT=true + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + return + elif [ "$level" == "WARN" ]; then + _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true + WARN_ALERT=true + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + return + elif [ "$level" == "NOTICE" ]; then + if [ "$_LOGGER_ERR_ONLY" != true ]; then + _Logger "$prefix$value" "$prefix$value" + fi + return + elif [ "$level" == "VERBOSE" ]; then + if [ $_LOGGER_VERBOSE == true ]; then + _Logger "$prefix($level):$value" "$prefix$value" + fi + return + elif [ "$level" == "ALWAYS" ]; then + _Logger "$prefix$value" "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "$prefix$value" "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return + else + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true + fi +} +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -186,10 +347,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -238,12 +423,12 @@ function SetLocalOSSettings { esac if [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "BusyBox" ]; then - QuickLogger "Cannot be installed on [$LOCAL_OS]. Please use $PROGRAM.sh directly." + Logger "Cannot be installed on [$LOCAL_OS]. Please use $PROGRAM.sh directly." "SIMPLE" exit 1 fi if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then - QuickLogger "Must be run as $USER." + Logger "Must be run as $USER." "SIMPLE" exit 1 fi @@ -251,14 +436,19 @@ function SetLocalOSSettings { } function GetInit { - if [ -f /sbin/init ]; then + if [ -f /sbin/openrc-run ]; then + init="openrc" + Logger "Detected openrc." "SIMPLE" + elif [ -f /sbin/init ]; then if file /sbin/init | grep systemd > /dev/null; then init="systemd" + Logger "Detected systemd." "SIMPLE" else init="initV" + Logger "Detected initV." "SIMPLE" fi else - QuickLogger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron." + Logger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron." "SIMPLE" init="none" fi } @@ -269,9 +459,9 @@ function CreateDir { if [ ! -d "$dir" ]; then mkdir -p "$dir" if [ $? == 0 ]; then - QuickLogger "Created directory [$dir]." + Logger "Created directory [$dir]." "SIMPLE" else - QuickLogger "Cannot create directory [$dir]." + Logger "Cannot create directory [$dir]." "SIMPLE" exit 1 fi fi @@ -280,36 +470,39 @@ function CreateDir { function CopyFile { local sourcePath="${1}" local destPath="${2}" - local fileName="${3}" - local fileMod="${4}" - local fileUser="${5}" - local fileGroup="${6}" - local overwrite="${7:-false}" + local sourceFileName="${3}" + local destFileName="${4}" + local fileMod="${5}" + local fileUser="${6}" + local fileGroup="${7}" + local overwrite="${8:-false}" local userGroup="" local oldFileName - if [ -f "$destPath/$fileName" ] && [ $overwrite == false ]; then - oldFileName="$fileName" - fileName="$oldFileName.new" - cp "$sourcePath/$oldFileName" "$destPath/$fileName" - else - cp "$sourcePath/$fileName" "$destPath" + if [ "$destFileName" == "" ]; then + destFileName="$sourceFileName" fi + if [ -f "$destPath/$destFileName" ] && [ $overwrite == false ]; then + destfileName="$sourceFileName.new" + Logger "Copying [$sourceFileName] to [$destPath/$destFilename]." "SIMPLE" + fi + + cp "$sourcePath/$sourceFileName" "$destPath/$destFileName" if [ $? != 0 ]; then - QuickLogger "Cannot copy [$fileName] to [$destPath]. Make sure to run install script in the directory containing all other files." - QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]." + Logger "Cannot copy [$sourcePath/$sourceFileName] to [$destPath/$destFileName]. Make sure to run install script in the directory containing all other files." "SIMPLE" + Logger "Also make sure you have permissions to write to [$BIN_DIR]." "SIMPLE" exit 1 else - QuickLogger "Copied [$fileName] to [$destPath]." + Logger "Copied [$sourcePath/$sourceFileName] to [$destPath/$destFileName]." "SIMPLE" if [ "$fileMod" != "" ]; then - chmod "$fileMod" "$destPath/$fileName" + chmod "$fileMod" "$destPath/$destFileName" if [ $? != 0 ]; then - QuickLogger "Cannot set file permissions of [$destPath/$fileName] to [$fileMod]." + Logger "Cannot set file permissions of [$destPath/$destFileName] to [$fileMod]." "SIMPLE" exit 1 else - QuickLogger "Set file permissions to [$fileMod] on [$destPath/$fileName]." + Logger "Set file permissions to [$fileMod] on [$destPath/$destFileName]." "SIMPLE" fi fi @@ -320,12 +513,12 @@ function CopyFile { userGroup="$userGroup"":$fileGroup" fi - chown "$userGroup" "$destPath/$fileName" + chown "$userGroup" "$destPath/$destFileName" if [ $? != 0 ]; then - QuickLogger "Could not set file ownership on [$destPath/$fileName] to [$userGroup]." + Logger "Could not set file ownership on [$destPath/$destFileName] to [$userGroup]." "SIMPLE" exit 1 else - QuickLogger "Set file ownership on [$destPath/$fileName] to [$userGroup]." + Logger "Set file ownership on [$destPath/$destFileName] to [$userGroup]." "SIMPLE" fi fi fi @@ -341,7 +534,7 @@ function CopyExampleFiles { for file in "${exampleFiles[@]}"; do if [ -f "$SCRIPT_PATH/$file" ]; then - CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "" "" "" false + CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "$file" "" "" "" false fi done } @@ -365,32 +558,38 @@ function CopyProgram { fi for file in "${binFiles[@]}"; do - CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" true + CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" "$file" 755 "$user" "$group" true done } function CopyServiceFiles { if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then CreateDir "$SERVICE_DIR_SYSTEMD_SYSTEM" - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_USER" ]; then CreateDir "$SERVICE_DIR_SYSTEMD_USER" - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true fi - QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." - QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." - QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." - QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]." + Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." "SIMPLE" + Logger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." "SIMPLE" + Logger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." "SIMPLE" + Logger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]." "SIMPLE" elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then - CreateDir "$SERVICE_DIR_INIT" - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true + #CreateDir "$SERVICE_DIR_INIT" + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "$SERVICE_FILE_INIT" "755" "" "" true - QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." - QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]." - QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]." + Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." "SIMPLE" + Logger "Can be activated with [service $SERVICE_FILE_INIT start]." "SIMPLE" + Logger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]." "SIMPLE" + elif ([ "$init" == "openrc" && [ -f "$SCRIPT_PATH/$SERVICE_FILE_OPENRC" ] && [ -d "$SERVICE_DIR_OPENRC" ]); then + # Rename service to usual service file + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_OPENRC" "$SERVICE_FILE_OPENRC" "$SERVICE_FILE_INIT" "755" "" "" true + + Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_OPENRC]." "SIMPLE" + Logger "Can be activated with [rc-update add $SERVICE_NAME.instance] where instance is a configuration file found in /etc/osync." "SIMPLE" else - QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation." + Logger "Cannot define what init style is in use on this system. Skipping service file installation." "SIMPLE" fi } @@ -409,7 +608,7 @@ function Statistics { fi fi - QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please." + Logger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please." "SIMPLE" return 1 } @@ -419,12 +618,12 @@ function RemoveFile { if [ -f "$file" ]; then rm -f "$file" if [ $? != 0 ]; then - QuickLogger "Could not remove file [$file]." + Logger "Could not remove file [$file]." "SIMPLE" else - QuickLogger "Removed file [$file]." + Logger "Removed file [$file]." "SIMPLE" fi else - QuickLogger "File [$file] not found. Skipping." + Logger "File [$file] not found. Skipping." "SIMPLE" fi } @@ -438,13 +637,13 @@ function RemoveAll { if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal RemoveFile "$BIN_DIR/$SSH_FILTER" else - QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it." + Logger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it." "SIMPLE" fi RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT" - QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually." + Logger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually." "SIMPLE" } function Usage { @@ -465,7 +664,7 @@ STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSI if [ "$ACTION" == "uninstall" ]; then RemoveAll - QuickLogger "$PROGRAM uninstalled." + Logger "$PROGRAM uninstalled." "SIMPLE" else CreateDir "$CONF_DIR" CreateDir "$BIN_DIR" @@ -474,11 +673,11 @@ else if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then CopyServiceFiles fi - QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM" + Logger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM" "SIMPLE" if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then - QuickLogger "" - QuickLogger "If connecting remotely, consider setup ssh filter to enhance security." - QuickLogger "" + echo "" + Logger "If connecting remotely, consider setup ssh filter to enhance security." "SIMPLE" + echo "" fi fi @@ -486,7 +685,7 @@ if [ $_STATS -eq 1 ]; then if [ $_LOGGER_SILENT == true ]; then Statistics else - QuickLogger "In order to make usage statistics, the script would like to connect to $STATS_LINK" + Logger "In order to make usage statistics, the script would like to connect to $STATS_LINK" "SIMPLE" read -r -p "No data except those in the url will be send. Allow [Y/n] " response case $response in [nN]) diff --git a/osync.sh b/osync.sh index b45e103..d95fe72 100755 --- a/osync.sh +++ b/osync.sh @@ -8,21 +8,19 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2.5-dev -PROGRAM_BUILD=2018032201 +PROGRAM_BUILD=2018062506 IS_STABLE=no +#TODO: tidy up ExecTasks comments + #TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" -#done: add checkRFC function (and use it for --destination-mails) -#done: ExecTasks still needs some better call argument list -#done: ExecTasks sub function relocate -#done: SendMail and SendEmail convert functions inverted, check on osync and obackup #command line arguments don't take -AaqV for example _OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018031501 +_OFUNCTIONS_BUILD=2018062504 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -85,9 +83,6 @@ fi SCRIPT_PID=$$ -# TODO: Check if %N works on MacOS -TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') - LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) @@ -115,6 +110,46 @@ else RUN_DIR=. fi +#### PoorMansRandomGenerator SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + + local minimum=1 + local maximum + local n=0 + + if [ "$digits" == "" ]; then + digits=5 + fi + + # Minimum already has a digit + for n in $(seq 1 $((digits-1))); do + minimum=$minimum"0" + maximum=$maximum"9" + done + maximum=$maximum"9" + + #n=0; while [ $n -lt $minimum ]; do n=$n$(dd if=/dev/urandom bs=100 count=1 2>/dev/null | tr -cd '0-9'); done; n=$(echo $n | sed -e 's/^0//') + # bs=19 since if real random strikes, having a 19 digits number is not supported + while [ $n -lt $minimum ] || [ $n -gt $maximum ]; do + if [ $n -lt $minimum ]; then + # Add numbers + n=$n$(dd if=/dev/urandom bs=19 count=1 2>/dev/null | tr -cd '0-9') + n=$(echo $n | sed -e 's/^0//') + if [ "$n" == "" ]; then + n=0 + fi + elif [ $n -gt $maximum ]; then + n=$(echo $n | sed 's/.$//') + fi + done + echo $n +} +#### PoorMansRandomGenerator SUBSET END #### + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) # Default alert attachment filename ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" @@ -129,7 +164,6 @@ function Dummy { sleep $SLEEP_TIME } -#### Logger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -145,8 +179,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -231,6 +268,7 @@ function RemoteLogger { # VERBOSE sent to stdout if _LOGGER_VERBOSE = true # ALWAYS is sent to stdout unless _LOGGER_SILENT = true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level @@ -282,35 +320,18 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } -#### Logger SUBSET END #### - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { @@ -324,7 +345,7 @@ function KillChilds { fi if kill -0 "$pid" > /dev/null 2>&1; then - # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment + #TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" @@ -966,11 +987,11 @@ function ExecTasks { else # pid is dead, get its exit code from wait command wait $pid - retval=$? + retval=$? #TODO: do we use retval codes somehow ?? where # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then - Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi @@ -1202,25 +1223,23 @@ function IsNumeric { fi } -# Checks email address validity -function CheckRFC822 { - local mail="${1}" - local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - - if [[ $mail =~ $rfc822 ]]; then - echo 1 - else - echo 0 - fi -} - +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -1255,12 +1274,24 @@ function HumanToNumeric { echo $value } -## from https://gist.github.com/cdown/1163649 +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} + +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -1363,10 +1394,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -1390,6 +1445,8 @@ function GetOs { local localOsVar local localOsName local localOsVer + local localOsBitness + local localOsFamily local osInfo="/etc/os-release" @@ -1416,9 +1473,36 @@ function GetOs { localOsName="${localOsName##*=}" localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - echo "$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" } GetOs @@ -2331,13 +2415,23 @@ function TrapError { (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } # Converts human readable sizes into integer kilobyte sizes @@ -2385,8 +2479,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -2676,13 +2773,23 @@ function ArrayContains () { echo 0 return } +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -2700,8 +2807,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3659,8 +3769,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3873,6 +3986,34 @@ function deletionPropagation { fi } +function Initialize { + + Logger "Initializing initiator and target file lists." "NOTICE" + + treeList "${INITIATOR[$__replicaDir]}" "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__type]}${INITIATOR[$__treeAfterFile]}" & + initiatorPid="$!" + + treeList "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${TARGET[$__type]}${INITIATOR[$__treeAfterFile]}" & + targetPid="$!" + + ExecTasks "$initiatorPid;$targetPid" "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING + if [ $? -ne 0 ]; then + IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")" + initiatorFail=false + targetFail=false + for pid in "${pidArray[@]}"; do + pid=${pid%:*} + if [ "$pid" == "$initiatorPid" ]; then + Logger "Failed to create initialization files for initiator." "ERROR" + elif [ "$pid" == "$targetPid" ]; then + Logger "Failed to create initialization files for target." "ERROR" + fi + done + exit 1 + resumeTarget="${SYNC_ACTION[8]}" + fi +} + ###### Sync function in 9 steps ###### ###### Step 0a & 0b: Create current file list of replicas @@ -4472,13 +4613,23 @@ function TrapError { (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") fi } +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } # Converts human readable sizes into integer kilobyte sizes @@ -4526,8 +4677,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -4692,7 +4846,7 @@ function SoftDelete { fi } -function _SummaryFromFile { +function _SummaryFromRsyncFile { local replicaPath="${1}" local summaryFile="${2}" local direction="${3}" @@ -4709,6 +4863,19 @@ function _SummaryFromFile { fi } +function _SummaryFromDeleteFile { + local replicaPath="${1}" + local summaryFile="${2}" + local direction="${3}" + + + if [ -f "$summaryFile" ]; then + while read -r file; do + Logger "$direction $replicaPath$file" "ALWAYS" + done < "$summaryFile" + fi +} + function Summary { ( @@ -4716,20 +4883,20 @@ function Summary { Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS" - _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID.$TSTAMP" "~ >>" - _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID.$TSTAMP" "~ <<" + _SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID.$TSTAMP" "~ >>" + _SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID.$TSTAMP" "~ <<" - Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS" - _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID.$TSTAMP" "+ >>" - _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID.$TSTAMP" "+ <<" + Logger "File transfers: INITIATOR << >> TARGET (may include file ownership and timestamp attributes)" "ALWAYS" + _SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID.$TSTAMP" "+ >>" + _SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID.$TSTAMP" "+ <<" Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS" if [ "$REMOTE_OPERATION" == "yes" ]; then - _SummaryFromFile "${TARGET[$__replicaDir]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" "- >>" + _SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" "- >>" else - _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID.$TSTAMP" "- >>" + _SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID.$TSTAMP" "- >>" fi - _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID.$TSTAMP" "- <<" + _SummaryFromDeleteFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID.$TSTAMP" "- <<" ) } @@ -4738,9 +4905,13 @@ function LogConflicts { local subject local body + # We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX + if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then + Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" + fi + ( _LOGGER_PREFIX="" - Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" if [ -f "$RUN_DIR/$PROGRAM.conflictList.comapre.$SCRIPT_PID.$TSTAMP" ]; then echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" while read -r line; do @@ -4994,6 +5165,7 @@ function Usage { echo "--force-unlock Will override any existing active or dead locks on initiator and target replica" echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead" echo "--no-resume Do not try to resume a failed run. By default, execution is resumed once" + echo "--initialize Create file lists without actually synchronizing anything, this will help setup deletion detections before the first run" echo "" echo "[QUICKSYNC OPTIONS]" @@ -5106,7 +5278,7 @@ sync_on_changes=false _NOLOCKS=false osync_cmd=$0 _SUMMARY=false - +INITIALIZE="no" function GetCommandlineArguments { local isFirstArgument=true @@ -5204,6 +5376,10 @@ function GetCommandlineArguments { LOG_CONFLICTS="yes" opts=$opts" --alert-conflicts" ;; + --initialize) + INITIALIZE="yes" + opts=$opts "--initialize" + ;; --no-prefix) opts=$opts" --no-prefix" _LOGGER_PREFIX="" @@ -5261,10 +5437,21 @@ if [ $_QUICK_SYNC -eq 2 ]; then HARD_MAX_EXEC_TIME=0 fi + if [ $(IsInteger $MAX_EXEC_TIME_PER_CMD_BEFORE) -ne 1 ]; then + MAX_EXEC_TIME_PER_CMD_BEFORE=0 + fi + + if [ $(IsInteger $MAX_EXEC_TIME_PER_CMD_AFTER) -ne 1 ]; then + MAX_EXEC_TIME_PER_CMD_AFTER=0 + fi + if [ "$PATH_SEPARATOR_CHAR" == "" ]; then PATH_SEPARATOR_CHAR=";" fi + + if [ $(IsInteger $MIN_WAIT) -ne 1 ]; then MIN_WAIT=30 + fi else ConfigFile="${1}" LoadConfigFile "$ConfigFile" @@ -5317,14 +5504,20 @@ else fi CheckReplicas RunBeforeHook - Main - if [ $? -eq 0 ]; then - SoftDelete - fi - if [ $_SUMMARY == true ]; then - Summary - fi - if [ $LOG_CONFLICTS == "yes" ]; then - LogConflicts + + if [ "$INITIALIZE" == "yes" ]; then + HandleLocks + Initialize + else + Main + if [ $? -eq 0 ]; then + SoftDelete + fi + if [ $_SUMMARY == true ]; then + Summary + fi + if [ $LOG_CONFLICTS == "yes" ]; then + LogConflicts + fi fi fi diff --git a/osync_target_helper.sh b/osync_target_helper.sh index b822380..ba61bc9 100755 --- a/osync_target_helper.sh +++ b/osync_target_helper.sh @@ -9,14 +9,10 @@ IS_STABLE=no #TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" -#done: add checkRFC function (and use it for --destination-mails) -#done: ExecTasks still needs some better call argument list -#done: ExecTasks sub function relocate -#done: SendMail and SendEmail convert functions inverted, check on osync and obackup #command line arguments don't take -AaqV for example _OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018031501 +_OFUNCTIONS_BUILD=2018062504 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -79,9 +75,6 @@ fi SCRIPT_PID=$$ -# TODO: Check if %N works on MacOS -TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') - LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) @@ -109,6 +102,46 @@ else RUN_DIR=. fi +#### PoorMansRandomGenerator SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + + local minimum=1 + local maximum + local n=0 + + if [ "$digits" == "" ]; then + digits=5 + fi + + # Minimum already has a digit + for n in $(seq 1 $((digits-1))); do + minimum=$minimum"0" + maximum=$maximum"9" + done + maximum=$maximum"9" + + #n=0; while [ $n -lt $minimum ]; do n=$n$(dd if=/dev/urandom bs=100 count=1 2>/dev/null | tr -cd '0-9'); done; n=$(echo $n | sed -e 's/^0//') + # bs=19 since if real random strikes, having a 19 digits number is not supported + while [ $n -lt $minimum ] || [ $n -gt $maximum ]; do + if [ $n -lt $minimum ]; then + # Add numbers + n=$n$(dd if=/dev/urandom bs=19 count=1 2>/dev/null | tr -cd '0-9') + n=$(echo $n | sed -e 's/^0//') + if [ "$n" == "" ]; then + n=0 + fi + elif [ $n -gt $maximum ]; then + n=$(echo $n | sed 's/.$//') + fi + done + echo $n +} +#### PoorMansRandomGenerator SUBSET END #### + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) # Default alert attachment filename ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" @@ -123,7 +156,6 @@ function Dummy { sleep $SLEEP_TIME } -#### Logger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -139,8 +171,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -225,6 +260,7 @@ function RemoteLogger { # VERBOSE sent to stdout if _LOGGER_VERBOSE = true # ALWAYS is sent to stdout unless _LOGGER_SILENT = true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level @@ -276,35 +312,18 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } -#### Logger SUBSET END #### - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { @@ -318,7 +337,7 @@ function KillChilds { fi if kill -0 "$pid" > /dev/null 2>&1; then - # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment + #TODO: Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" @@ -960,11 +979,11 @@ function ExecTasks { else # pid is dead, get its exit code from wait command wait $pid - retval=$? + retval=$? #TODO: do we use retval codes somehow ?? where # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then - Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" #TODO: set this to debug in order to stop complaints if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi @@ -1196,25 +1215,23 @@ function IsNumeric { fi } -# Checks email address validity -function CheckRFC822 { - local mail="${1}" - local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - - if [[ $mail =~ $rfc822 ]]; then - echo 1 - else - echo 0 - fi -} - +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -1249,12 +1266,24 @@ function HumanToNumeric { echo $value } -## from https://gist.github.com/cdown/1163649 +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} + +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -1357,10 +1386,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -1384,6 +1437,8 @@ function GetOs { local localOsVar local localOsName local localOsVer + local localOsBitness + local localOsFamily local osInfo="/etc/os-release" @@ -1410,9 +1465,36 @@ function GetOs { localOsName="${localOsName##*=}" localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - echo "$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" } GetOs @@ -2228,8 +2310,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then