From b7f21f2d895db0b7a8806efa6bbf0743f310f6bc Mon Sep 17 00:00:00 2001
From: Shadowigor <shadowigorr@gmail.com>
Date: Mon, 29 Aug 2016 01:33:21 +0200
Subject: [PATCH] Updated auto-generated scripts

---
 dev/debug_osync.sh | 360 ++++++++++++++++++++++++++++++---------------
 osync.sh           | 250 ++++++++++++++++++++++---------
 2 files changed, 427 insertions(+), 183 deletions(-)

diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh
index 6d0c483..29093fe 100755
--- a/dev/debug_osync.sh
+++ b/dev/debug_osync.sh
@@ -1,56 +1,60 @@
 #!/usr/bin/env bash
 
 #TODO(critical): handle conflict prevalance, especially in sync_attrs function
-#TODO(critical): writelockfiles remote does not shut execution
 
 PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
 AUTHOR="(C) 2013-2016 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
 PROGRAM_VERSION=1.2-dev-parallel
-PROGRAM_BUILD=2016082201
+PROGRAM_BUILD=2016082803
 IS_STABLE=no
 
-#	Function Name		Is parallel	#__WITH_PARANOIA_DEBUG
+# Execution order
+#	Function Name				Is parallel	#__WITH_PARANOIA_DEBUG
 
-#	GetLocalOS		no		#__WITH_PARANOIA_DEBUG
-#	InitLocalOSSettings	no		#__WITH_PARANOIA_DEBUG
-#	CheckEnvironment	no		#__WITH_PARANOIA_DEBUG
-#	PreInit			no		#__WITH_PARANOIA_DEBUG
-#	Init			no		#__WITH_PARANOIA_DEBUG
-#	PostInit		no		#__WITH_PARANOIA_DEBUG
-#	GetRemoteOS		no		#__WITH_PARANOIA_DEBUG
-#	InitRemoteOSSettings	no		#__WITH_PARANOIA_DEBUG
-#	CheckReplicaPaths	yes		#__WITH_PARANOIA_DEBUG
-#	CheckDiskSpace		yes		#__WITH_PARANOIA_DEBUG
-#	RunBeforeHook		yes		#__WITH_PARANOIA_DEBUG
-#	Main			no		#__WITH_PARANOIA_DEBUG
-#	CreateStateDirs		yes		#__WITH_PARANOIA_DEBUG
-#	CheckLocks		yes		#__WITH_PARANOIA_DEBUG
-#	WriteLockFiles		yes		#__WITH_PARANOIA_DEBUG
-#	Sync			no		#__WITH_PARANOIA_DEBUG
-#	tree_list		yes		#__WITH_PARANOIA_DEBUG
-#	tree_list		yes		#__WITH_PARANOIA_DEBUG
-#	delete_list		yes		#__WITH_PARANOIA_DEBUG
-#	delete_list		yes		#__WITH_PARANOIA_DEBUG
-#	sync_attrs		no		#__WITH_PARANOIA_DEBUG
-#	_get_file_ctime_mtime	yes		#__WITH_PARANOIA_DEBUG
-#	sync_update		yes		#__WITH_PARANOIA_DEBUG
-#	sync_update		yes		#__WITH_PARANOIA_DEBUG
-#	deletion_propagation	yes		#__WITH_PARANOIA_DEBUG
-#	deletion_propagation	yes		#__WITH_PARANOIA_DEBUG
-#	tree_list		yes		#__WITH_PARANOIA_DEBUG
-#	tree_list		yes		#__WITH_PARANOIA_DEBUG
-#	SoftDelete		yes		#__WITH_PARANOIA_DEBUG
-#	RunAfterHook		yes		#__WITH_PARANOIA_DEBUG
-#	UnlockReplicas		yes		#__WITH_PARANOIA_DEBUG
-#	CleanUp			no		#__WITH_PARANOIA_DEBUG
+#	GetLocalOS				no		#__WITH_PARANOIA_DEBUG
+#	InitLocalOSSettings			no		#__WITH_PARANOIA_DEBUG
+#	CheckEnvironment			no		#__WITH_PARANOIA_DEBUG
+#	PreInit					no		#__WITH_PARANOIA_DEBUG
+#	Init					no		#__WITH_PARANOIA_DEBUG
+#	PostInit				no		#__WITH_PARANOIA_DEBUG
+#	GetRemoteOS				no		#__WITH_PARANOIA_DEBUG
+#	InitRemoteOSSettings			no		#__WITH_PARANOIA_DEBUG
+#	CheckReplicaPaths			yes		#__WITH_PARANOIA_DEBUG
+#	CheckDiskSpace				yes		#__WITH_PARANOIA_DEBUG
+#	RunBeforeHook				yes		#__WITH_PARANOIA_DEBUG
+#	Main					no		#__WITH_PARANOIA_DEBUG
+#		CreateStateDirs			yes		#__WITH_PARANOIA_DEBUG
+#	 	CheckLocks			yes		#__WITH_PARANOIA_DEBUG
+#	 	WriteLockFiles			yes		#__WITH_PARANOIA_DEBUG
+#	 	Sync				no		#__WITH_PARANOIA_DEBUG
+#			tree_list		yes		#__WITH_PARANOIA_DEBUG
+#			tree_list		yes		#__WITH_PARANOIA_DEBUG
+#			delete_list		yes		#__WITH_PARANOIA_DEBUG
+#			delete_list		yes		#__WITH_PARANOIA_DEBUG
+#			sync_attrs		no		#__WITH_PARANOIA_DEBUG
+#			_get_file_ctime_mtime	yes		#__WITH_PARANOIA_DEBUG
+#			sync_update		yes		#__WITH_PARANOIA_DEBUG
+#			sync_update		yes		#__WITH_PARANOIA_DEBUG
+#			deletion_propagation	yes		#__WITH_PARANOIA_DEBUG
+#			deletion_propagation	yes		#__WITH_PARANOIA_DEBUG
+#			tree_list		yes		#__WITH_PARANOIA_DEBUG
+#			tree_list		yes		#__WITH_PARANOIA_DEBUG
+#		SoftDelete			yes		#__WITH_PARANOIA_DEBUG
+#	RunAfterHook				yes		#__WITH_PARANOIA_DEBUG
+#	UnlockReplicas				yes		#__WITH_PARANOIA_DEBUG
+#	CleanUp					no		#__WITH_PARANOIA_DEBUG
 
 #### MINIMAL-FUNCTION-SET BEGIN ####
 
-## FUNC_BUILD=2016082204
-## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
+## FUNC_BUILD=2016082802
+## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
+
+## To use in a program, define the following variables:
+## PROGRAM=program-name
+## INSTANCE_ID=program-instance-name
+## _DEBUG=yes/no
 
-## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
 if ! type "$BASH" > /dev/null; then
 	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
 	exit 127
@@ -75,6 +79,9 @@ fi
 ERROR_ALERT=0
 WARN_ALERT=0
 
+# Current log
+CURRENT_LOG=
+
 ## allow function call checks			#__WITH_PARANOIA_DEBUG
 if [ "$_PARANOIA_DEBUG" == "yes" ];then		#__WITH_PARANOIA_DEBUG
 	_DEBUG=yes				#__WITH_PARANOIA_DEBUG
@@ -96,6 +103,10 @@ SCRIPT_PID=$$
 LOCAL_USER=$(whoami)
 LOCAL_HOST=$(hostname)
 
+if [ "$PROGRAM" == "" ]; then
+	PROGRAM="ofunctions"
+fi
+
 ## Default log file until config file is loaded
 if [ -w /var/log ]; then
 	LOG_FILE="/var/log/$PROGRAM.log"
@@ -136,6 +147,7 @@ function _Logger {
 	local evalue="${3}" # What to log to stderr
 
 	echo -e "$lvalue" >> "$LOG_FILE"
+	CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
 
 	if [ "$_LOGGER_STDERR" -eq 1 ]; then
 		cat <<< "$evalue" 1>&2
@@ -267,11 +279,14 @@ function KillAllChilds {
 
 # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
 function SendAlert {
-	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
+	local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
+
+	__CheckArguments 0-1 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
 	local mail_no_attachment=
 	local attachment_command=
 	local subject=
+	local body=
 
 	# Windows specific settings
 	local encryption_string=
@@ -300,7 +315,8 @@ function SendAlert {
 	else
 		mail_no_attachment=0
 	fi
-	MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
+	body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
+
 	if [ $ERROR_ALERT -eq 1 ]; then
 		subject="Error alert for $INSTANCE_ID"
 	elif [ $WARN_ALERT -eq 1 ]; then
@@ -309,11 +325,17 @@ function SendAlert {
 		subject="Alert for $INSTANCE_ID"
 	fi
 
+	if [ $runAlert == true ]; then
+		subject="Currently runing - $subject"
+	else
+		subject="Fnished run - $subject"
+	fi
+
 	if [ "$mail_no_attachment" -eq 0 ]; then
 		attachment_command="-a $ALERT_LOG_FILE"
 	fi
 	if type mutt > /dev/null 2>&1 ; then
-		echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
+		echo "$body" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN"
 		else
@@ -330,10 +352,10 @@ function SendAlert {
 		else
 			attachment_command=""
 		fi
-		echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
+		echo "$body" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN"
-			echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
+			echo "$body" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
 			if [ $? != 0 ]; then
 				Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN"
 			else
@@ -347,7 +369,7 @@ function SendAlert {
 	fi
 
 	if type sendmail > /dev/null 2>&1 ; then
-		echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
+		echo -e "Subject:$subject\r\n$body" | $(type -p sendmail) $DESTINATION_MAILS
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
 		else
@@ -370,7 +392,7 @@ function SendAlert {
 		if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
 			auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
 		fi
-		$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
+		$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$body" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
 		if [ $? != 0 ]; then
 			Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
 		else
@@ -386,7 +408,7 @@ function SendAlert {
 		else
 			SMTP_OPTIONS=""
 		fi
-		$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
+		$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$body" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN"
 		else
@@ -397,7 +419,7 @@ function SendAlert {
 
 	# pfSense specific
 	if [ -f /usr/local/bin/mail.php ]; then
-		echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject"
+		echo "$body" | /usr/local/bin/mail.php -s="$subject"
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
 		else
@@ -616,6 +638,7 @@ function joinString {
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
 # Warning: Don't imbricate this function into another run if you plan to use the global variable output
+
 function WaitForTaskCompletion {
 	local pids="${1}" # pids to wait for, separated by semi-colon
 	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
@@ -636,9 +659,13 @@ function WaitForTaskCompletion {
 	local retval=0 # return value of monitored pid process
 	local errorcount=0 # Number of pids that finished with errors
 
+	local pid	# Current pid working on
 	local pidCount # number of given pids
 	local pidState # State of the process
 
+	local pidsArray # Array of currently running pids
+	local newPidsArray # New array of currently running pids
+
 	IFS=';' read -a pidsArray <<< "$pids"
 	pidCount=${#pidsArray[@]}
 
@@ -667,7 +694,7 @@ function WaitForTaskCompletion {
 			if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
 				Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
 				soft_alert=1
-				SendAlert
+				SendAlert true
 
 			fi
 			if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
@@ -680,7 +707,7 @@ function WaitForTaskCompletion {
 						Logger "Could not stop task with pid [$pid]." "ERROR"
 					fi
 				done
-				SendAlert
+				SendAlert true
 				errrorcount=$((errorcount+1))
 			fi
 		fi
@@ -699,17 +726,18 @@ function WaitForTaskCompletion {
 				retval=$?
 				if [ $retval -ne 0 ]; then
 					errorcount=$((errorcount+1))
-					Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
+					Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
 					if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then
-						WAIT_FOR_TASK_COMPLETION="$pid:$result"
+						WAIT_FOR_TASK_COMPLETION="$pid:$retval"
 					else
-						WAIT_FOR_TASK_COMPLETION=";$pid:$result"
+						WAIT_FOR_TASK_COMPLETION=";$pid:$retval"
 					fi
 				fi
 			fi
 		done
 
 		pidsArray=("${newPidsArray[@]}")
+		# Trivial wait time for bash to not eat up all CPU
 		sleep $SLEEP_TIME
 	done
 
@@ -723,6 +751,66 @@ function WaitForTaskCompletion {
 	fi
 }
 
+# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
+# Returns the number of non zero exit codes from commands
+function ParallelExec {
+	local numberOfProcesses="${1}" # Number of simultaneous commands to run
+	local commandsArg="${2}" # Semi-colon separated list of commands
+
+	local pid
+	local runningPids=0
+	local counter=0
+	local commandsArray
+	local pidsArray
+	local newPidsArray
+	local retval
+	local retvalAll=0
+	local pidState
+	local commandsArrayPid
+
+	IFS=';' read -r -a commandsArray <<< "$commandsArg"
+
+	Logger "Runnning ${#commandsArray[@]} commands in $numberOfProcesses simultaneous processes." "DEBUG"
+
+	while [ $counter -lt "${#commandsArray[@]}" ] || [ ${#pidsArray[@]} -gt 0 ]; do
+
+		while [ $counter -lt "${#commandsArray[@]}" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
+			Logger "Running command [${commandsArray[$counter]}]." "DEBUG"
+			eval "${commandsArray[$counter]}" &
+			pid=$!
+			pidsArray+=($pid)
+			commandsArrayPid[$pid]="${commandsArray[$counter]}"
+			counter=$((counter+1))
+		done
+
+
+		newPidsArray=()
+		for pid in "${pidsArray[@]}"; do
+			# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
+			if kill -0 $pid > /dev/null 2>&1; then
+				pidState=$(ps -p$pid -o state= 2 > /dev/null)
+				if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
+					newPidsArray+=($pid)
+				fi
+			else
+				# pid is dead, get it's exit code from wait command
+				wait $pid
+				retval=$?
+				if [ $retval -ne 0 ]; then
+					Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
+					retvalAll=$((retvalAll+1))
+				fi
+			fi
+		done
+		pidsArray=("${newPidsArray[@]}")
+
+		# Trivial wait time for bash to not eat up all CPU
+		sleep $SLEEP_TIME
+	done
+
+	return $retvalAll
+}
+
 function CleanUp {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
@@ -1016,7 +1104,7 @@ function CheckConnectivityRemoteHost {
 
 		if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
 			eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
-			WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} true $KEEP_LOGGING
+			WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING
 			if [ $? != 0 ]; then
 				Logger "Cannot ping $REMOTE_HOST" "ERROR"
 				return 1
@@ -1061,12 +1149,15 @@ function __CheckArguments {
 	# Checks the number of arguments of a function and raises an error if some are missing
 
 	if [ "$_DEBUG" == "yes" ]; then
-		local number_of_arguments="${1}" # Number of arguments the tested function should have
-		local number_of_given_arguments="${2}" # Number of arguments that have been passed
-		local function_name="${3}" # Function name that called __CheckArguments
+		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
+		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
+		local functionName="${3}" # Function name that called __CheckArguments
+
+		local minArgs
+		local maxArgs
 
 		if [ "$_PARANOIA_DEBUG" == "yes" ]; then
-			Logger "Entering function [$function_name]." "DEBUG"
+			Logger "Entering function [$functionName]." "DEBUG"
 		fi
 
 		# All arguments of the function to check are passed as array in ${4} (the function call waits for $@)
@@ -1074,23 +1165,31 @@ function __CheckArguments {
 		# In order to avoid this, we need to iterate over ${4} and count
 
 		local iterate=4
-		local fetch_arguments=1
-		local arg_list=""
-		while [ $fetch_arguments -eq 1 ]; do
+		local fetchArguments=1
+		local argList=""
+		local countedArguments
+		while [ $fetchArguments -eq 1 ]; do
 			cmd='argument=${'$iterate'}'
 			eval $cmd
 			if [ "$argument" = "" ]; then
-				fetch_arguments=0
+				fetchArguments=0
 			else
-				arg_list="$arg_list [Argument $(($iterate-3)): $argument]"
+				argList="$arg_list [Argument $(($iterate-3)): $argument]"
 				iterate=$(($iterate+1))
 			fi
 		done
-		local counted_arguments=$((iterate-4))
+		countedArguments=$((iterate-4))
 
-		if [ $counted_arguments -ne $number_of_arguments ]; then
-			Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR"
-			Logger "Arguments passed: $arg_list" "ERROR"
+		if [ $(IsNumeric "$numberOfArguments") -eq 1 ]; then
+			minArgs=$numberOfArguments
+			maxArgs=$numberOfArguments
+		else
+			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
+		fi
+
+		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
+			Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR"
+			Logger "Arguments passed: $argList" "ERROR"
 		fi
 	fi
 }
@@ -1334,7 +1433,6 @@ function InitLocalOSSettings {
 function InitRemoteOSSettings {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"    #__WITH_PARANOIA_DEBUG
 
-	#TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes
 	## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
 	if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
 		if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
@@ -1448,6 +1546,24 @@ function CheckEnvironment {
 function CheckCurrentConfig {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"    #__WITH_PARANOIA_DEBUG
 
+	# Check all variables that should contain "yes" or "no"
+	declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
+	for i in "${yes_no_vars[@]}"; do
+		test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
+		eval "$test"
+	done
+
+	# Check all variables that should contain a numerical value >= 0
+	declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
+	for i in "${num_vars[@]}"; do
+		test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
+		eval "$test"
+	done
+}
+
+function CheckCurrentConfigAll {
+	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"    #__WITH_PARANOIA_DEBUG
+
 	if [ "$INSTANCE_ID" == "" ]; then
 		Logger "No INSTANCE_ID defined in config file." "CRITICAL"
 		exit 1
@@ -1463,21 +1579,11 @@ function CheckCurrentConfig {
 		exit 1
 	fi
 
-	# Check all variables that should contain "yes" or "no"
-	declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
-	for i in "${yes_no_vars[@]}"; do
-		test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
-		eval "$test"
-	done
-
-	# Check all variables that should contain a numerical value >= 0
-	declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
-	for i in "${num_vars[@]}"; do
-		test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
-		eval "$test"
-	done
-
 	#TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
+	if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then
+		Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL"
+		exit 1
+	fi
 }
 
 ###### Osync specific functions (non shared)
@@ -1792,22 +1898,24 @@ function CheckLocks {
 
 function _WriteLockFilesLocal {
 	local lockfile="${1}"
-	__CheckArguments 1 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
+	local replica_type="${2}"
+	__CheckArguments 2 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
 	$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile"
 	if [ $?	!= 0 ]; then
-		Logger "Could not create lock file [$lockfile]." "CRITICAL"
+		Logger "Could not create lock file on local $replica_type in [$lockfile]." "CRITICAL"
 		exit 1
 	else
-		Logger "Locked replica on [$lockfile]." "DEBUG"
+		Logger "Locked local $replica_type replica in [$lockfile]." "DEBUG"
 	fi
 }
 
 function _WriteLockFilesRemote {
 	local lockfile="${1}"
-	__CheckArguments 1 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
+	local replica_type="${2}"
+	__CheckArguments 2 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
-	local cmd=
+	local cmd
 
 	CheckConnectivity3rdPartyHosts
 	CheckConnectivityRemoteHost
@@ -1816,33 +1924,49 @@ function _WriteLockFilesRemote {
 	Logger "cmd: $cmd" "DEBUG"
 	eval "$cmd"
 	if [ $? != 0 ]; then
-		Logger "Could not set lock on remote $replica_type replica." "CRITICAL"
+		Logger "Could not create lock file on remote $replica_type in [$lockfile]." "CRITICAL"
 		exit 1
 	else
-		Logger "Locked remote $replica_type replica." "DEBUG"
+		Logger "Locked remote $replica_type replica in [$lockfile]." "DEBUG"
 	fi
 }
 
 function WriteLockFiles {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
-	local pids
+	local initiatorPid
+	local targetPid
+	local pidArray
+	local pid
+
+	_WriteLockFilesLocal "${INITIATOR[2]}" "${INITIATOR[0]}"&
+	initiatorPid="$!"
 
-	_WriteLockFilesLocal "${INITIATOR[2]}" &
-	pids="$!"
 	if [ "$REMOTE_OPERATION" != "yes" ]; then
-		_WriteLockFilesLocal "${TARGET[2]}" &
-		pids="$pids;$!"
+		_WriteLockFilesLocal "${TARGET[2]}" "${TARGET[0]}" &
+		targetPid="$!"
 	else
-		_WriteLockFilesRemote "${TARGET[2]}" &
-		pids="$pids;$!"
+		_WriteLockFilesRemote "${TARGET[2]}" "${TARGET[0]}" &
+		targetPid="$!"
 	fi
-	WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
+
+	INITIATOR_LOCK_FILE_EXISTS=true
+	TARGET_LOCK_FILE_EXISTS=true
+	WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
 	if [ $? -ne 0 ]; then
+		IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
+		for pid in "${pidArray[@]}"; do
+			pid=${pid%:*}
+			if [ $pid == $initiatorPid ]; then
+				INITIATOR_LOCK_FILE_EXISTS=false
+			elif [ $pid == $targetPid ]; then
+				TARGET_LOCK_FILE_EXISTS=false
+			fi
+		done
+
 		Logger "Cancelling task." "CRITICAL"
 		exit 1
 	fi
-	LOCK_FILES_EXIST=1
 }
 
 function _UnlockReplicasLocal {
@@ -1884,21 +2008,29 @@ function UnlockReplicas {
 
 	local pids
 
-	if [ $_NOLOCKS -eq 1 ] || [ $LOCK_FILES_EXIST -eq 0 ]; then
+	if [ $_NOLOCKS -eq 1 ]; then
 		return 0
 	fi
 
-	_UnlockReplicasLocal "${INITIATOR[2]}" &
-	pids="$!"
-
-	if [ "$REMOTE_OPERATION" != "yes" ]; then
-		_UnlockReplicasLocal "${TARGET[2]}" &
-		pids="$pids;$!"
-	else
-		_UnlockReplicasRemote "${TARGET[2]}" &
-		pids="$pids;$!"
+	if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then
+		_UnlockReplicasLocal "${INITIATOR[2]}" &
+		pids="$!"
+	fi
+
+	#WIP check if WaitForTaskCompletion can handle emppty pids like ";pid"
+	if [ $TARGET_LOCK_FILE_EXISTS == true ]; then
+		if [ "$REMOTE_OPERATION" != "yes" ]; then
+			_UnlockReplicasLocal "${TARGET[2]}" &
+			pids="$pids;$!"
+		else
+			_UnlockReplicasRemote "${TARGET[2]}" &
+			pids="$pids;$!"
+		fi
+	fi
+
+	if [ "$pids" != "" ]; then
+		WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
 	fi
-	WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
 }
 
 ###### Sync core functions
@@ -1918,11 +2050,6 @@ function tree_list {
 
 	__CheckArguments 3 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
-	#WIP
-	#if [ "$replica_type" == "${INITIATOR[0]}" ]; then
-	#	exit 12
-	#fi
-
 	escaped_replica_path=$(EscapeSpaces "$replica_path")
 
 	Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
@@ -2039,7 +2166,6 @@ function sync_attrs {
 	Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
 	eval "$rsync_cmd"
 	WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
-	#WIP: return retval from process instead of err count if only one pid is tested
 	retval=$?
 	if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
 		Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
@@ -2120,7 +2246,6 @@ function sync_attrs {
 	Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
 	eval "$rsync_cmd"
 	WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
-	#WIP: the same
 	retval=$?
 	if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
 		Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
@@ -3133,7 +3258,9 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then
 	CONFLICT_PREVALANCE=initiator
 fi
 
-LOCK_FILES_EXIST=0
+INITIATOR_LOCK_FILE_EXISTS=false
+TARGET_LOCK_FILE_EXISTS=false
+
 FORCE_UNLOCK=0
 no_maxtime=0
 opts=""
@@ -3291,6 +3418,7 @@ opts="${opts# *}"
 	if [ $_QUICK_SYNC -lt 2 ]; then
 		CheckCurrentConfig
 	fi
+	CheckCurrentConfigAll
 
 	DATE=$(date)
 	Logger "-------------------------------------------------------------" "NOTICE"
diff --git a/osync.sh b/osync.sh
index fd69046..dfd89f0 100755
--- a/osync.sh
+++ b/osync.sh
@@ -1,23 +1,27 @@
 #!/usr/bin/env bash
 
 #TODO(critical): handle conflict prevalance, especially in sync_attrs function
-#TODO(critical): writelockfiles remote does not shut execution
 
 PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
 AUTHOR="(C) 2013-2016 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
 PROGRAM_VERSION=1.2-dev-parallel
-PROGRAM_BUILD=2016082201
+PROGRAM_BUILD=2016082803
 IS_STABLE=no
 
+# Execution order
 
 
 #### MINIMAL-FUNCTION-SET BEGIN ####
 
-## FUNC_BUILD=2016082204
-## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
+## FUNC_BUILD=2016082802
+## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
+
+## To use in a program, define the following variables:
+## PROGRAM=program-name
+## INSTANCE_ID=program-instance-name
+## _DEBUG=yes/no
 
-## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
 if ! type "$BASH" > /dev/null; then
 	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
 	exit 127
@@ -42,6 +46,9 @@ fi
 ERROR_ALERT=0
 WARN_ALERT=0
 
+# Current log
+CURRENT_LOG=
+
 
 ## allow debugging from command line with _DEBUG=yes
 if [ ! "$_DEBUG" == "yes" ]; then
@@ -59,6 +66,10 @@ SCRIPT_PID=$$
 LOCAL_USER=$(whoami)
 LOCAL_HOST=$(hostname)
 
+if [ "$PROGRAM" == "" ]; then
+	PROGRAM="ofunctions"
+fi
+
 ## Default log file until config file is loaded
 if [ -w /var/log ]; then
 	LOG_FILE="/var/log/$PROGRAM.log"
@@ -98,6 +109,7 @@ function _Logger {
 	local evalue="${3}" # What to log to stderr
 
 	echo -e "$lvalue" >> "$LOG_FILE"
+	CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
 
 	if [ "$_LOGGER_STDERR" -eq 1 ]; then
 		cat <<< "$evalue" 1>&2
@@ -220,10 +232,13 @@ function KillAllChilds {
 
 # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
 function SendAlert {
+	local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
+
 
 	local mail_no_attachment=
 	local attachment_command=
 	local subject=
+	local body=
 
 	# Windows specific settings
 	local encryption_string=
@@ -252,7 +267,8 @@ function SendAlert {
 	else
 		mail_no_attachment=0
 	fi
-	MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
+	body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
+
 	if [ $ERROR_ALERT -eq 1 ]; then
 		subject="Error alert for $INSTANCE_ID"
 	elif [ $WARN_ALERT -eq 1 ]; then
@@ -261,11 +277,17 @@ function SendAlert {
 		subject="Alert for $INSTANCE_ID"
 	fi
 
+	if [ $runAlert == true ]; then
+		subject="Currently runing - $subject"
+	else
+		subject="Fnished run - $subject"
+	fi
+
 	if [ "$mail_no_attachment" -eq 0 ]; then
 		attachment_command="-a $ALERT_LOG_FILE"
 	fi
 	if type mutt > /dev/null 2>&1 ; then
-		echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
+		echo "$body" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN"
 		else
@@ -282,10 +304,10 @@ function SendAlert {
 		else
 			attachment_command=""
 		fi
-		echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
+		echo "$body" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN"
-			echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
+			echo "$body" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
 			if [ $? != 0 ]; then
 				Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN"
 			else
@@ -299,7 +321,7 @@ function SendAlert {
 	fi
 
 	if type sendmail > /dev/null 2>&1 ; then
-		echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
+		echo -e "Subject:$subject\r\n$body" | $(type -p sendmail) $DESTINATION_MAILS
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
 		else
@@ -322,7 +344,7 @@ function SendAlert {
 		if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
 			auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
 		fi
-		$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
+		$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$body" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
 		if [ $? != 0 ]; then
 			Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
 		else
@@ -338,7 +360,7 @@ function SendAlert {
 		else
 			SMTP_OPTIONS=""
 		fi
-		$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
+		$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$body" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN"
 		else
@@ -349,7 +371,7 @@ function SendAlert {
 
 	# pfSense specific
 	if [ -f /usr/local/bin/mail.php ]; then
-		echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject"
+		echo "$body" | /usr/local/bin/mail.php -s="$subject"
 		if [ $? != 0 ]; then
 			Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
 		else
@@ -566,6 +588,7 @@ function joinString {
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
 # Warning: Don't imbricate this function into another run if you plan to use the global variable output
+
 function WaitForTaskCompletion {
 	local pids="${1}" # pids to wait for, separated by semi-colon
 	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
@@ -584,9 +607,13 @@ function WaitForTaskCompletion {
 	local retval=0 # return value of monitored pid process
 	local errorcount=0 # Number of pids that finished with errors
 
+	local pid	# Current pid working on
 	local pidCount # number of given pids
 	local pidState # State of the process
 
+	local pidsArray # Array of currently running pids
+	local newPidsArray # New array of currently running pids
+
 	IFS=';' read -a pidsArray <<< "$pids"
 	pidCount=${#pidsArray[@]}
 
@@ -615,7 +642,7 @@ function WaitForTaskCompletion {
 			if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
 				Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
 				soft_alert=1
-				SendAlert
+				SendAlert true
 
 			fi
 			if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
@@ -628,7 +655,7 @@ function WaitForTaskCompletion {
 						Logger "Could not stop task with pid [$pid]." "ERROR"
 					fi
 				done
-				SendAlert
+				SendAlert true
 				errrorcount=$((errorcount+1))
 			fi
 		fi
@@ -647,17 +674,18 @@ function WaitForTaskCompletion {
 				retval=$?
 				if [ $retval -ne 0 ]; then
 					errorcount=$((errorcount+1))
-					Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
+					Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
 					if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then
-						WAIT_FOR_TASK_COMPLETION="$pid:$result"
+						WAIT_FOR_TASK_COMPLETION="$pid:$retval"
 					else
-						WAIT_FOR_TASK_COMPLETION=";$pid:$result"
+						WAIT_FOR_TASK_COMPLETION=";$pid:$retval"
 					fi
 				fi
 			fi
 		done
 
 		pidsArray=("${newPidsArray[@]}")
+		# Trivial wait time for bash to not eat up all CPU
 		sleep $SLEEP_TIME
 	done
 
@@ -670,6 +698,66 @@ function WaitForTaskCompletion {
 	fi
 }
 
+# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
+# Returns the number of non zero exit codes from commands
+function ParallelExec {
+	local numberOfProcesses="${1}" # Number of simultaneous commands to run
+	local commandsArg="${2}" # Semi-colon separated list of commands
+
+	local pid
+	local runningPids=0
+	local counter=0
+	local commandsArray
+	local pidsArray
+	local newPidsArray
+	local retval
+	local retvalAll=0
+	local pidState
+	local commandsArrayPid
+
+	IFS=';' read -r -a commandsArray <<< "$commandsArg"
+
+	Logger "Runnning ${#commandsArray[@]} commands in $numberOfProcesses simultaneous processes." "DEBUG"
+
+	while [ $counter -lt "${#commandsArray[@]}" ] || [ ${#pidsArray[@]} -gt 0 ]; do
+
+		while [ $counter -lt "${#commandsArray[@]}" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
+			Logger "Running command [${commandsArray[$counter]}]." "DEBUG"
+			eval "${commandsArray[$counter]}" &
+			pid=$!
+			pidsArray+=($pid)
+			commandsArrayPid[$pid]="${commandsArray[$counter]}"
+			counter=$((counter+1))
+		done
+
+
+		newPidsArray=()
+		for pid in "${pidsArray[@]}"; do
+			# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
+			if kill -0 $pid > /dev/null 2>&1; then
+				pidState=$(ps -p$pid -o state= 2 > /dev/null)
+				if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
+					newPidsArray+=($pid)
+				fi
+			else
+				# pid is dead, get it's exit code from wait command
+				wait $pid
+				retval=$?
+				if [ $retval -ne 0 ]; then
+					Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
+					retvalAll=$((retvalAll+1))
+				fi
+			fi
+		done
+		pidsArray=("${newPidsArray[@]}")
+
+		# Trivial wait time for bash to not eat up all CPU
+		sleep $SLEEP_TIME
+	done
+
+	return $retvalAll
+}
+
 function CleanUp {
 
 	if [ "$_DEBUG" != "yes" ]; then
@@ -955,7 +1043,7 @@ function CheckConnectivityRemoteHost {
 
 		if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
 			eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
-			WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} true $KEEP_LOGGING
+			WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING
 			if [ $? != 0 ]; then
 				Logger "Cannot ping $REMOTE_HOST" "ERROR"
 				return 1
@@ -1227,7 +1315,6 @@ function InitLocalOSSettings {
 
 function InitRemoteOSSettings {
 
-	#TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes
 	## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
 	if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
 		if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
@@ -1339,6 +1426,23 @@ function CheckEnvironment {
 
 function CheckCurrentConfig {
 
+	# Check all variables that should contain "yes" or "no"
+	declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
+	for i in "${yes_no_vars[@]}"; do
+		test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
+		eval "$test"
+	done
+
+	# Check all variables that should contain a numerical value >= 0
+	declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
+	for i in "${num_vars[@]}"; do
+		test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
+		eval "$test"
+	done
+}
+
+function CheckCurrentConfigAll {
+
 	if [ "$INSTANCE_ID" == "" ]; then
 		Logger "No INSTANCE_ID defined in config file." "CRITICAL"
 		exit 1
@@ -1354,21 +1458,11 @@ function CheckCurrentConfig {
 		exit 1
 	fi
 
-	# Check all variables that should contain "yes" or "no"
-	declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
-	for i in "${yes_no_vars[@]}"; do
-		test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
-		eval "$test"
-	done
-
-	# Check all variables that should contain a numerical value >= 0
-	declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
-	for i in "${num_vars[@]}"; do
-		test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
-		eval "$test"
-	done
-
 	#TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
+	if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then
+		Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL"
+		exit 1
+	fi
 }
 
 ###### Osync specific functions (non shared)
@@ -1671,20 +1765,22 @@ function CheckLocks {
 
 function _WriteLockFilesLocal {
 	local lockfile="${1}"
+	local replica_type="${2}"
 
 	$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile"
 	if [ $?	!= 0 ]; then
-		Logger "Could not create lock file [$lockfile]." "CRITICAL"
+		Logger "Could not create lock file on local $replica_type in [$lockfile]." "CRITICAL"
 		exit 1
 	else
-		Logger "Locked replica on [$lockfile]." "DEBUG"
+		Logger "Locked local $replica_type replica in [$lockfile]." "DEBUG"
 	fi
 }
 
 function _WriteLockFilesRemote {
 	local lockfile="${1}"
+	local replica_type="${2}"
 
-	local cmd=
+	local cmd
 
 	CheckConnectivity3rdPartyHosts
 	CheckConnectivityRemoteHost
@@ -1693,32 +1789,48 @@ function _WriteLockFilesRemote {
 	Logger "cmd: $cmd" "DEBUG"
 	eval "$cmd"
 	if [ $? != 0 ]; then
-		Logger "Could not set lock on remote $replica_type replica." "CRITICAL"
+		Logger "Could not create lock file on remote $replica_type in [$lockfile]." "CRITICAL"
 		exit 1
 	else
-		Logger "Locked remote $replica_type replica." "DEBUG"
+		Logger "Locked remote $replica_type replica in [$lockfile]." "DEBUG"
 	fi
 }
 
 function WriteLockFiles {
 
-	local pids
+	local initiatorPid
+	local targetPid
+	local pidArray
+	local pid
+
+	_WriteLockFilesLocal "${INITIATOR[2]}" "${INITIATOR[0]}"&
+	initiatorPid="$!"
 
-	_WriteLockFilesLocal "${INITIATOR[2]}" &
-	pids="$!"
 	if [ "$REMOTE_OPERATION" != "yes" ]; then
-		_WriteLockFilesLocal "${TARGET[2]}" &
-		pids="$pids;$!"
+		_WriteLockFilesLocal "${TARGET[2]}" "${TARGET[0]}" &
+		targetPid="$!"
 	else
-		_WriteLockFilesRemote "${TARGET[2]}" &
-		pids="$pids;$!"
+		_WriteLockFilesRemote "${TARGET[2]}" "${TARGET[0]}" &
+		targetPid="$!"
 	fi
-	WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
+
+	INITIATOR_LOCK_FILE_EXISTS=true
+	TARGET_LOCK_FILE_EXISTS=true
+	WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
 	if [ $? -ne 0 ]; then
+		IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
+		for pid in "${pidArray[@]}"; do
+			pid=${pid%:*}
+			if [ $pid == $initiatorPid ]; then
+				INITIATOR_LOCK_FILE_EXISTS=false
+			elif [ $pid == $targetPid ]; then
+				TARGET_LOCK_FILE_EXISTS=false
+			fi
+		done
+
 		Logger "Cancelling task." "CRITICAL"
 		exit 1
 	fi
-	LOCK_FILES_EXIST=1
 }
 
 function _UnlockReplicasLocal {
@@ -1757,21 +1869,29 @@ function UnlockReplicas {
 
 	local pids
 
-	if [ $_NOLOCKS -eq 1 ] || [ $LOCK_FILES_EXIST -eq 0 ]; then
+	if [ $_NOLOCKS -eq 1 ]; then
 		return 0
 	fi
 
-	_UnlockReplicasLocal "${INITIATOR[2]}" &
-	pids="$!"
-
-	if [ "$REMOTE_OPERATION" != "yes" ]; then
-		_UnlockReplicasLocal "${TARGET[2]}" &
-		pids="$pids;$!"
-	else
-		_UnlockReplicasRemote "${TARGET[2]}" &
-		pids="$pids;$!"
+	if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then
+		_UnlockReplicasLocal "${INITIATOR[2]}" &
+		pids="$!"
+	fi
+
+	#WIP check if WaitForTaskCompletion can handle emppty pids like ";pid"
+	if [ $TARGET_LOCK_FILE_EXISTS == true ]; then
+		if [ "$REMOTE_OPERATION" != "yes" ]; then
+			_UnlockReplicasLocal "${TARGET[2]}" &
+			pids="$pids;$!"
+		else
+			_UnlockReplicasRemote "${TARGET[2]}" &
+			pids="$pids;$!"
+		fi
+	fi
+
+	if [ "$pids" != "" ]; then
+		WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
 	fi
-	WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
 }
 
 ###### Sync core functions
@@ -1790,11 +1910,6 @@ function tree_list {
 	local rsync_cmd
 
 
-	#WIP
-	#if [ "$replica_type" == "${INITIATOR[0]}" ]; then
-	#	exit 12
-	#fi
-
 	escaped_replica_path=$(EscapeSpaces "$replica_path")
 
 	Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
@@ -1907,7 +2022,6 @@ function sync_attrs {
 	Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
 	eval "$rsync_cmd"
 	WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
-	#WIP: return retval from process instead of err count if only one pid is tested
 	retval=$?
 	if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
 		Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
@@ -1988,7 +2102,6 @@ function sync_attrs {
 	Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
 	eval "$rsync_cmd"
 	WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
-	#WIP: the same
 	retval=$?
 	if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
 		Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
@@ -2989,7 +3102,9 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then
 	CONFLICT_PREVALANCE=initiator
 fi
 
-LOCK_FILES_EXIST=0
+INITIATOR_LOCK_FILE_EXISTS=false
+TARGET_LOCK_FILE_EXISTS=false
+
 FORCE_UNLOCK=0
 no_maxtime=0
 opts=""
@@ -3147,6 +3262,7 @@ opts="${opts# *}"
 	if [ $_QUICK_SYNC -lt 2 ]; then
 		CheckCurrentConfig
 	fi
+	CheckCurrentConfigAll
 
 	DATE=$(date)
 	Logger "-------------------------------------------------------------" "NOTICE"