A whole bunch of functionnality added.

osync.sh

@@ -5,15 +5,13 @@
 
 
 ## todo:
-# add dryrun
+# add dryrun, DOC: never run dry when already run real or you will lose your deleted file history
 # add logging
 # add resume on error
-# add master/slave prevalance
-# thread sync function
 # remote functionnality
 
 OSYNC_VERSION=0.4
-OSYNC_BUILD=1607201302
+OSYNC_BUILD=1607201305
 
 DEBUG=no
 SCRIPT_PID=$$
@@ -32,10 +30,10 @@ KEEP_LOGGING=1801
 
 function Log
 {
-        echo "TIME: $SECONDS - $1" >> "$LOG_FILE"
+        echo -e "TIME: $SECONDS - $1" >> "$LOG_FILE"
         if [ $silent -eq 0 ]
         then
-                echo "TIME: $SECONDS - $1"
+                echo -e "TIME: $SECONDS - $1"
         fi
 }
 
@@ -56,6 +54,23 @@ function TrapError
         fi
 }
 
+function TrapStop
+{
+        LogError " /!\ WARNING: Manual exit of osync is really not recommended. Sync will be in inconsistent state."
+	LogError " /!\ WARNING: If you are sure, please hit CTRL+C another time to quit."
+	if [ $soft_stop -eq 0 ]
+	then
+		soft_stop=1
+	fi
+
+        if [ "$DEBUG" == "no" ] && [ $soft_stop -eq 1 ]
+        then
+                CleanUp
+		exit 1
+        fi
+}
+
+
 function Spinner
 {
         if [ $silent -eq 1 ]
@@ -96,6 +111,14 @@ function CleanUp
         rm -f /dev/shm/osync_config_$SCRIPT_PID
  	rm -f /dev/shm/osync_run_local_$SCRIPT_PID
 	rm -f /dev/shm/osync_run_remote_$SCRIPT_PID
+	rm -f /dev/shm/osync_master-tree-current_$SCRIPT_PID
+	rm -f /dev/shm/osync_slave-tree-current_$SCRIPT_PID
+	rm -f /dev/shm/osync_master-tree-before_$SCRIPT_PID
+	rm -f /dev/shm/osync_slave-tree-before_$SCRIPT_PID
+	rm -f /dev/shm/osync_update_master_replica_$SCRIPT_PID
+	rm -f /dev/shm/osync_update_slave_replica_$SCRIPT_PID
+	rm -f /dev/shm/osync_deletition_on_master_$SCRIPT_PID
+	rm -f /dev/shm/osync_deletition_on_slave_$SCRIPT_PID
 }
 
 function SendAlert
@@ -381,14 +404,44 @@ function LockSlave
 function sync_update_master
 {
 	Log "Updating slave replica."
-	rsync -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
+	rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
+	child_pid=$!
+       	WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
+        retval=$?
+        if [ $retval != 0 ]
+        then
+                LogError "Updating slave replica failed. Stopping execution."
+        else
+                Log "Updating slave replica succeded."
+        fi
+	
+	if [ "$VERBOSE_LOGS" == "yes" ]
+	then
+		Log "List:\n$(cat /dev/shm/osync_update_slave_replica_$SCRIPT_PID)"
+	fi
+	return $retval
 }
 
 # Subfunction of Sync
 function sync_update_slave
 {
 	Log "Updating master replica."
-	rsync -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
+	rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
+	child_pid=$!
+        WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
+        retval=$?
+        if [ $retval != 0 ]
+        then
+                LogError "Updating master replica failed. Stopping execution."
+        else
+                Log "Updating master replica succeded."
+        fi
+
+	if [ "$VERBOSE_LOGS" == "yes" ]
+	then
+		Log "List:\n$(cat /dev/shm/osync_update_master_replica_$SCRIPT_PID)"
+	fi
+	return $retval
 }
 
 function Sync
@@ -398,14 +451,32 @@ function Sync
 
 	Log "Starting synchronization task."
 
-	## Create local file list
 	Log "Creating master replica file list."
-	rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-current
+	rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_master-tree-current_$SCRIPT_PID &
-	Log "Creating slave replica file list."
+	child_pid=$!
-	## Create distant file list
+	WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
-	rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-current
+	retval=$?
+	if [ $retval == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ]
+	then
+		mv /dev/shm/osync_master-tree-current_$SCRIPT_PID $STATE_DIR/master-tree-current
+	else
+		LogError "Cannot create master file list."
+		return 1
+	fi
+
+	Log "Creating slave replica file list."
+	rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
+	child_pid=$!
+	WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
+	retval=$?
+	if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-current_$SCRIPT_PID ]
+	then
+		mv /dev/shm/osync_slave-tree-current_$SCRIPT_PID $STATE_DIR/slave-tree-current
+	else
+		LogError "Cannot create slave file list."
+		return 1
+	fi
 
-	## diff local file list and before file list except if before file list is empty >> deleted
 	Log "Creating master replica deleted file list."
 	if [ -f $STATE_DIR/master-tree-before ]
 	then
@@ -414,7 +485,6 @@ function Sync
 		touch $STATE_DIR/master-deleted-list
 	fi
 
-	## diff local file list and before file list except if before file list is empty >> deleted
 	Log "Creating slave replica deleted file list."
 	if [ -f $STATE_DIR/slave-tree-before ]
 	then
@@ -426,26 +496,83 @@ function Sync
 	if [ "$CONFLICT_PREVALANCE" != "master" ]
 	then
 		sync_update_slave
+		if [ $? != 0 ]
+		then
+			return 1
+		fi
 		sync_update_master
+		if [ $? != 0 ]
+		then
+			return 1
+		fi
 	else
 		sync_update_master
+		if [ $? != 0 ]
+		then
+			return 1
+		fi
 		sync_update_slave
+		if [ $? != 0 ]
+		then
+			return 1
+		fi
 	fi
 
 	Log "Propagating deletitions to slave replica."
-	rsync -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ 
+	rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --include-from "$STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
+	child_pid=$!
+	WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
+	if [ $? != 0 ]
+	then
+		LogError "Deletition on slave failed."
+		return 1
+	fi
+	if [ "$VERBOSE_LOGS" == "yes" ]
+	then
+		Log "List:\n$(cat /dev/shm/osync_deletition_on_slave_$SCRIPT_PID)"
+	fi
+	
 	Log "Propagating deletitions to master replica."
-	rsync -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ 
+	rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --include-from "$STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
+	child_pid=$!
+	WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
+	if [ $? != 0 ]
+	then
+		LogError "Deletition on master failed."
+		return 1
+	fi
+	if [ "$VERBOSE_LOGS" == "yes" ]
+	then
+		Log "List:\n$(cat /dev/shm/osync_deletition_on_master_$SCRIPT_PID)"
+	fi
 
-        ## Create local file list
         Log "Creating new master replica file list."
-        rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-before
+        rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_master-tree-before_$SCRIPT_PID &
+	child_pid=$!
+        WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
+        retval=$?
+        if [ $retval == 0 ] && [ -f /dev/shm/osync_master-tree-before_$SCRIPT_PID ]
+        then
+                mv /dev/shm/osync_master-tree-before_$SCRIPT_PID $STATE_DIR/master-tree-before
+        else
+                LogError "Cannot create slave file list."
+                return 1
+        fi
+
         Log "Creating new slave replica file list."
-        ## Create distant file list
+        rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_slave-tree-before_$SCRIPT_PID &
-        rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-before
+	child_pid=$!
+        WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
+        retval=$?
+        if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-before_$SCRIPT_PID ]
+        then
+                mv /dev/shm/osync_slave-tree-before_$SCRIPT_PID $STATE_DIR/slave-tree-before
+        else
+                LogError "Cannot create slave file list."
+                return 1
+        fi
 
 	Log "Finished synchronization task."
-
 }
 
 function SoftDelete
@@ -511,6 +638,13 @@ function Init
 	else
 		RUNNER=""
 	fi
+
+	## Dryrun option
+	if [ $dryrun -eq 1 ]
+	then
+		DRY_OPTION=--dry-run
+		DRY_WARNING="/!\ DRY RUN"
+	fi
 }
 
 function Main
@@ -536,6 +670,7 @@ silent=0
 # Alert flags
 soft_alert_total=0
 error_alert=0
+soft_stop=0
 
 if [ $# -eq 0 ]
 then
@@ -569,7 +704,7 @@ then
 			Init
 			DATE=$(date)
 			Log "-----------------------------------------------------------"
-			Log "$DATE - Osync v$OSYNC_VERSION script begin."
+			Log " $DRY_WARNING $DATE - Osync v$OSYNC_VERSION script begin."
 			Log "-----------------------------------------------------------"
 			CheckMasterSlaveDirs
 			if [ $? == 0 ]

sync.conf

@@ -2,14 +2,14 @@
 
 ###### Osync file synchronizer tool based on rsync
 ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
-#### Config file rev 1507201301
+#### Config file rev 1607201301
 
 ## Sync job identification, any string you want
 SYNC_ID="sync_test"
 
 ## Directories to synchronize
-MASTER_SYNC_DIR="/home/git/osync/test/dir1"
+MASTER_SYNC_DIR="/root/backups"
-SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
+SLAVE_SYNC_DIR="/srv/DONNEES/sl"
 
 ## Generate an alert if master or slave have lass space than given value in KB.
 MINIMUM_SPACE=1048576
@@ -45,6 +45,8 @@ CONFLICT_BACKUP_DAYS=30
 SOFT_DELETE=yes
 SOFT_DELETE_DAYS=30
 
+VERBOSE_LOGS=yes
+
 DESTINATION_MAILS="ozy@badministrateur.com"