Rebuilt targets

dev/debug_osync.sh

@@ -40,8 +40,8 @@ IS_STABLE=yes
 #	CleanUp					no		#__WITH_PARANOIA_DEBUG
 
 
-_OFUNCTIONS_VERSION=2.1
+_OFUNCTIONS_VERSION=2.1.2
-_OFUNCTIONS_BUILD=2017032301
+_OFUNCTIONS_BUILD=2017052601
 _OFUNCTIONS_BOOTSTRAP=true
 
 ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@@ -66,6 +66,9 @@ fi
 ## Correct output of sort command (language agnostic sorting)
 export LC_ALL=C
 
+## Default umask for file creation
+umask 0077
+
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
@@ -293,7 +296,7 @@ function Logger {
 		return
 	elif [ "$level" == "VERBOSE" ]; then
 		if [ $_LOGGER_VERBOSE == true ]; then
-			_Logger "$prefix:$value" "$prefix$value"
+			_Logger "$prefix($level):$value" "$prefix$value"
 		fi
 		return
 	elif [ "$level" == "ALWAYS" ]; then
@@ -662,6 +665,18 @@ function Spinner {
 	fi
 }
 
+function _PerfProfiler {									#__WITH_PARANOIA_DEBUG
+	local perfString									#__WITH_PARANOIA_DEBUG
+												#__WITH_PARANOIA_DEBUG
+	perfString=$(ps -p $$ -o %cpu,%mem,cmd,time)						#__WITH_PARANOIA_DEBUG
+												#__WITH_PARANOIA_DEBUG
+	for i in $(pgrep -P $$); do								#__WITH_PARANOIA_DEBUG
+		perfString="$perfString\n"$(ps -p $i -o %cpu,%mem,cmd,time | tail -1)		#__WITH_PARANOIA_DEBUG
+	done											#__WITH_PARANOIA_DEBUG
+												#__WITH_PARANOIA_DEBUG
+	Logger "PerfProfiler: $perfString" "PARANOIA_DEBUG"					#__WITH_PARANOIA_DEBUG
+}												#__WITH_PARANOIA_DEBUG
+
 
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@@ -794,6 +809,11 @@ function WaitForTaskCompletion {
 		pidsArray=("${newPidsArray[@]}")
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+		if [ "$_PERF_PROFILER" == "yes" ]; then				##__WITH_PARANOIA_DEBUG
+			_PerfProfiler						##__WITH_PARANOIA_DEBUG
+		fi								##__WITH_PARANOIA_DEBUG
+
 	done
 
 	Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
@@ -959,6 +979,10 @@ function ParallelExec {
 
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+		if [ "$_PERF_PROFILER" == "yes" ]; then				##__WITH_PARANOIA_DEBUG
+			_PerfProfiler						##__WITH_PARANOIA_DEBUG
+		fi								##__WITH_PARANOIA_DEBUG
 	done
 
 	return $errorCount
@@ -1184,6 +1208,62 @@ function GetLocalOS {
 	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
 }
 
+#__BEGIN_WITH_PARANOIA_DEBUG
+function __CheckArguments {
+	# Checks the number of arguments of a function and raises an error if some are missing
+
+	if [ "$_DEBUG" == "yes" ]; then
+		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
+		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
+
+		local minArgs
+		local maxArgs
+
+		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
+		# If any of the arguments contains spaces, bash things there are two aguments
+		# In order to avoid this, we need to iterate over ${3} and count
+
+		callerName="${FUNCNAME[1]}"
+
+		local iterate=3
+		local fetchArguments=true
+		local argList=""
+		local countedArguments
+		while [ $fetchArguments == true ]; do
+			cmd='argument=${'$iterate'}'
+			eval $cmd
+			if [ "$argument" == "" ]; then
+				fetchArguments=false
+			else
+				argList="$argList[Argument $((iterate-2)): $argument] "
+				iterate=$((iterate+1))
+			fi
+		done
+
+		countedArguments=$((iterate-3))
+
+		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
+			minArgs=$numberOfArguments
+			maxArgs=$numberOfArguments
+		else
+			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
+		fi
+
+		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
+
+		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
+			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
+			Logger "$callerName arguments: $argList" "ERROR"
+		else
+			if [ ! -z "$argList" ]; then
+				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
+			fi
+		fi
+	fi
+}
+
+#__END_WITH_PARANOIA_DEBUG
+
 
 function GetRemoteOS {
 	__CheckArguments 0 $# "$@"	#__WITH_PARANOIA_DEBUG
@@ -1439,62 +1519,6 @@ function CheckConnectivity3rdPartyHosts {
 	fi											#__WITH_PARANOIA_DEBUG
 }
 
-#__BEGIN_WITH_PARANOIA_DEBUG
-function __CheckArguments {
-	# Checks the number of arguments of a function and raises an error if some are missing
-
-	if [ "$_DEBUG" == "yes" ]; then
-		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
-		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
-
-		local minArgs
-		local maxArgs
-
-		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
-		# If any of the arguments contains spaces, bash things there are two aguments
-		# In order to avoid this, we need to iterate over ${3} and count
-
-		callerName="${FUNCNAME[1]}"
-
-		local iterate=3
-		local fetchArguments=true
-		local argList=""
-		local countedArguments
-		while [ $fetchArguments == true ]; do
-			cmd='argument=${'$iterate'}'
-			eval $cmd
-			if [ "$argument" == "" ]; then
-				fetchArguments=false
-			else
-				argList="$argList[Argument $((iterate-2)): $argument] "
-				iterate=$((iterate+1))
-			fi
-		done
-
-		countedArguments=$((iterate-3))
-
-		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
-			minArgs=$numberOfArguments
-			maxArgs=$numberOfArguments
-		else
-			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
-		fi
-
-		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
-
-		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
-			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
-			Logger "$callerName arguments: $argList" "ERROR"
-		else
-			if [ ! -z "$argList" ]; then
-				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
-			fi
-		fi
-	fi
-}
-
-#__END_WITH_PARANOIA_DEBUG
-
 function RsyncPatternsAdd {
 	local patternType="${1}"	# exclude or include
 	local pattern="${2}"
@@ -1845,11 +1869,18 @@ function ParentPid {
 
 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
-vercomp () {
+function VerComp () {
+	if [ "$1" == "" ] || [ "$2" == "" ]; then
+		Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
+		return 1
+	fi
+
 	if [[ $1 == $2 ]]
 		then
-        return 0
+			echo 0
+		return
 	fi
+
 	local IFS=.
 	local i ver1=($1) ver2=($2)
 	# fill empty fields in ver1 with zeros
@@ -1866,14 +1897,18 @@ vercomp () {
 		fi
 		if ((10#${ver1[i]} > 10#${ver2[i]}))
 		then
-            return 1
+			echo 1
+			return
         	fi
         	if ((10#${ver1[i]} < 10#${ver2[i]}))
         	then
-            return 2
+			echo 2
+            		return
         	fi
     	done
-    return 0
+
+    	echo 0
+	return
 }
 
 function GetConfFileValue () {
@@ -1906,6 +1941,29 @@ function SetConfFileValue () {
         fi
 }
 
+# Function can replace [ -f /some/file* ] tests
+# Modified version of http://stackoverflow.com/a/6364244/2635443
+function WildcardFileExists () {
+        local file="${1}"
+        local exists=0
+
+        for f in $file; do
+                ## Check if the glob gets expanded to existing files.
+                ## If not, f here will be exactly the pattern above
+                ## and the exists test will evaluate to false.
+                if [ -e "$f" ]; then
+                        exists=1
+                        break
+                fi
+        done
+
+        if [ $exists -eq 1 ]; then
+                echo 1
+        else
+                echo 0
+        fi
+}
+
 # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
 _OFUNCTIONS_BOOTSTRAP=true
 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1

install.sh

@@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
 PROGRAM_BATCH=$PROGRAM"-batch.sh"
 SSH_FILTER="ssh_filter.sh"
 
-SCRIPT_BUILD=2017032101
+SCRIPT_BUILD=2017041701
 
 ## osync / obackup / pmocr / zsnap install script
 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@@ -342,9 +342,9 @@ function CopyServiceFiles {
 	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
 		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
 
-		QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
-		QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
+		QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
-		QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
+		QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
 	else
 		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
 	fi
@@ -386,7 +386,11 @@ function RemoveFile {
 
 function RemoveAll {
 	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
+
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
 		RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	fi
+
 	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
 		RemoveFile "$BIN_DIR/$SSH_FILTER"
 	else
@@ -447,7 +451,9 @@ else
 	CreateDir "$BIN_DIR"
 	CopyExampleFiles
 	CopyProgram
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
 		CopyServiceFiles
+	fi
 	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
 	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
 		QuickLogger ""

osync.sh

@@ -9,8 +9,8 @@ IS_STABLE=yes
 
 
 
-_OFUNCTIONS_VERSION=2.1
+_OFUNCTIONS_VERSION=2.1.2
-_OFUNCTIONS_BUILD=2017032301
+_OFUNCTIONS_BUILD=2017052601
 _OFUNCTIONS_BOOTSTRAP=true
 
 ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@@ -35,6 +35,9 @@ fi
 ## Correct output of sort command (language agnostic sorting)
 export LC_ALL=C
 
+## Default umask for file creation
+umask 0077
+
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
@@ -252,7 +255,7 @@ function Logger {
 		return
 	elif [ "$level" == "VERBOSE" ]; then
 		if [ $_LOGGER_VERBOSE == true ]; then
-			_Logger "$prefix:$value" "$prefix$value"
+			_Logger "$prefix($level):$value" "$prefix$value"
 		fi
 		return
 	elif [ "$level" == "ALWAYS" ]; then
@@ -612,6 +615,7 @@ function Spinner {
 }
 
 
+
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
 # Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
@@ -736,6 +740,8 @@ function WaitForTaskCompletion {
 		pidsArray=("${newPidsArray[@]}")
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+
 	done
 
 
@@ -894,6 +900,7 @@ function ParallelExec {
 
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
 	done
 
 	return $errorCount
@@ -1119,6 +1126,7 @@ function GetLocalOS {
 }
 
 
+
 function GetRemoteOS {
 
 	if [ "$REMOTE_OPERATION" != "yes" ]; then
@@ -1362,7 +1370,6 @@ function CheckConnectivity3rdPartyHosts {
 		fi
 }
 
-
 function RsyncPatternsAdd {
 	local patternType="${1}"	# exclude or include
 	local pattern="${2}"
@@ -1706,11 +1713,18 @@ function ParentPid {
 
 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
-vercomp () {
+function VerComp () {
+	if [ "$1" == "" ] || [ "$2" == "" ]; then
+		Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
+		return 1
+	fi
+
 	if [[ $1 == $2 ]]
 		then
-        return 0
+			echo 0
+		return
 	fi
+
 	local IFS=.
 	local i ver1=($1) ver2=($2)
 	# fill empty fields in ver1 with zeros
@@ -1727,14 +1741,18 @@ vercomp () {
 		fi
 		if ((10#${ver1[i]} > 10#${ver2[i]}))
 		then
-            return 1
+			echo 1
+			return
         	fi
         	if ((10#${ver1[i]} < 10#${ver2[i]}))
         	then
-            return 2
+			echo 2
+            		return
         	fi
     	done
-    return 0
+
+    	echo 0
+	return
 }
 
 function GetConfFileValue () {
@@ -1767,6 +1785,29 @@ function SetConfFileValue () {
         fi
 }
 
+# Function can replace [ -f /some/file* ] tests
+# Modified version of http://stackoverflow.com/a/6364244/2635443
+function WildcardFileExists () {
+        local file="${1}"
+        local exists=0
+
+        for f in $file; do
+                ## Check if the glob gets expanded to existing files.
+                ## If not, f here will be exactly the pattern above
+                ## and the exists test will evaluate to false.
+                if [ -e "$f" ]; then
+                        exists=1
+                        break
+                fi
+        done
+
+        if [ $exists -eq 1 ]; then
+                echo 1
+        else
+                echo 0
+        fi
+}
+
 # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
 _OFUNCTIONS_BOOTSTRAP=true
 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1