WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finished work on summary function

This commit is contained in:
deajan 2016-11-15 13:10:47 +01:00
parent 6cc1286fe9
commit ce7c712329
1 changed files with 51 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
dev/n_osync.sh
View File

@ -4,9 +4,12 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2 PROGRAM_VERSION=1.2-beta2
PROGRAM_BUILD=2016111404 PROGRAM_BUILD=2016111501
IS_STABLE=no IS_STABLE=no
#TODO: space after dry warning message to remove
#TODO: remove local unused vars (multiplier etc)
# Execution order #__WITH_PARANOIA_DEBUG # Execution order #__WITH_PARANOIA_DEBUG
# Function Name Is parallel #__WITH_PARANOIA_DEBUG # Function Name Is parallel #__WITH_PARANOIA_DEBUG
@ -974,6 +977,17 @@ function _deleteLocal {
local previousFile="" local previousFile=""
local result local result
local deletionListFromReplica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
deletionListFromReplica="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
deletionListFromReplica="${INITIATOR[$__type]}"
else
Logger "Bogus replicaType in [${FUNCNAME[0]}]." "CRITICAL"
exit 1
fi
if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then
$COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -1028,7 +1042,7 @@ function _deleteLocal {
fi fi
previousFile="$files" previousFile="$files"
fi fi
done < "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" done < "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}"
} }
function _deleteRemote { function _deleteRemote {
@ -1040,21 +1054,29 @@ function _deleteRemote {
local escDestDir local escDestDir
local rsyncCmd local rsyncCmd
local escSourceFile
local failedDeleteList local failedDeleteList
local successDeleteList local successDeleteList
# Check this local deletionListFromReplica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
deletionListFromReplica="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
deletionListFromReplica="${INITIATOR[$__type]}"
else
Logger "Bogus replicaType in [${FUNCNAME[0]}]." "CRITICAL"
exit 1
fi
failedDeleteList="$(EscapeSpaces ${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$replicaType${TARGET[$__failedDeletedListFile]})" failedDeleteList="$(EscapeSpaces ${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$replicaType${TARGET[$__failedDeletedListFile]})"
successDeleteList="$(EscapeSpaces ${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$replicaType${TARGET[$_successDeletedListFile]})" successDeleteList="$(EscapeSpaces ${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$replicaType${TARGET[$__successDeletedListFile]})"
## This is a special coded function. Need to redelcare local functions on remote host, passing all needed variables as escaped arguments to ssh command. ## This is a special coded function. Need to redelcare local functions on remote host, passing all needed variables as escaped arguments to ssh command.
## Anything beetween << ENDSSH and ENDSSH will be executed remotely ## Anything beetween << ENDSSH and ENDSSH will be executed remotely
# Additionnaly, we need to copy the deletetion list to the remote state folder # Additionnaly, we need to copy the deletetion list to the remote state folder
escDestDir="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}")" escDestDir="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}")"
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}\" $REMOTE_USER@$REMOTE_HOST:\"$escDestDir/\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}\" $REMOTE_USER@$REMOTE_HOST:\"$escDestDir/\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" 2>> "$LOG_FILE" eval "$rsyncCmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -1065,7 +1087,7 @@ function _deleteRemote {
exit 1 exit 1
fi fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETED_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETE_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1
## The following lines are executed remotely ## The following lines are executed remotely
function _logger { function _logger {
@ -1109,6 +1131,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
## Empty earlier failed delete list ## Empty earlier failed delete list
> "$FAILED_DELETE_LIST" > "$FAILED_DELETE_LIST"
> "$SUCCESS_DELETE_LIST"
parentdir= parentdir=
previousFile="" previousFile=""
@ -1171,17 +1194,12 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
done < "$FILE_LIST" done < "$FILE_LIST"
ENDSSH ENDSSH
#sleep 5
if [ -f "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then
Logger "Remote Deletion:\n$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "VERBOSE" Logger "Remote Deletion:\n$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "VERBOSE"
fi fi
## Copy back the deleted failed file list ## Copy back the deleted failed file list
#WIP remove if working rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
#escSourceFile="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/${INITIATOR[$__failedDeletedListFile]}")"
#rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$escSourceFile\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$FAILED_DELETE_LIST,$SUCCESS_DELETE_LIST}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" 2>> "$LOG_FILE" eval "$rsyncCmd" 2>> "$LOG_FILE"
result=$? result=$?
@ -1197,7 +1215,7 @@ ENDSSH
# delete_Propagation(replica type) # delete_Propagation(replica type)
function deletionPropagation { function deletionPropagation {
local replicaType="${1}" # Contains replica type: initiator, target local replicaType="${1}" # Contains replica type: initiator, target where to delete
__CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local replicaDir local replicaDir
@ -1205,14 +1223,12 @@ function deletionPropagation {
Logger "Propagating deletions to $replicaType replica." "NOTICE" Logger "Propagating deletions to $replicaType replica." "NOTICE"
#TODO: deletionPropagation replicaType = source replica whereas _deleteXxxxxx replicaType = dest replica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then
replicaDir="${INITIATOR[$__replicaDir]}" replicaDir="${INITIATOR[$__replicaDir]}"
deleteDir="${INITIATOR[$__deleteDir]}" deleteDir="${INITIATOR[$__deleteDir]}"
_deleteLocal "${TARGET[$__type]}" "$replicaDir" "$deleteDir" _deleteLocal "${INITIATOR[$__type]}" "$replicaDir" "$deleteDir"
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Deletion on $replicaType replica failed." "CRITICAL" Logger "Deletion on $replicaType replica failed." "CRITICAL"
@ -1227,9 +1243,9 @@ function deletionPropagation {
deleteDir="${TARGET[$__deleteDir]}" deleteDir="${TARGET[$__deleteDir]}"
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
_deleteRemote "${INITIATOR[$__type]}" "$replicaDir" "$deleteDir" _deleteRemote "${TARGET[$__type]}" "$replicaDir" "$deleteDir"
else else
_deleteLocal "${INITIATOR[$__type]}" "$replicaDir" "$deleteDir" _deleteLocal "${TARGET[$__type]}" "$replicaDir" "$deleteDir"
fi fi
retval=$? retval=$?
if [ $retval == 0 ]; then if [ $retval == 0 ]; then
@ -1413,6 +1429,7 @@ function Sync {
## Step 2 ## Step 2
if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
#TODO(high) only -X and -A (xattr & acl) should trigger this
if [ "$RSYNC_ATTR_ARGS" != "" ]; then if [ "$RSYNC_ATTR_ARGS" != "" ]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
@ -1473,12 +1490,12 @@ function Sync {
## Step 4a & 4b ## Step 4a & 4b
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ]; then
deletionPropagation "${TARGET[$__type]}" & deletionPropagation "${INITIATOR[$__type]}" &
initiatorPid="$!" initiatorPid="$!"
fi fi
if [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then if [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
deletionPropagation "${INITIATOR[$__type]}" & deletionPropagation "${TARGET[$__type]}" &
targetPid="$!" targetPid="$!"
fi fi
@ -1700,14 +1717,15 @@ function SoftDelete {
} }
function _SummaryFromFile { function _SummaryFromFile {
local summaryFile="${1}" local replicaPath="${1}"
local direction="${2}" local summaryFile="${2}"
local direction="${3}"
__CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
if [ -f "$summaryFile" ]; then if [ -f "$summaryFile" ]; then
while read -r file; do while read -r file; do
Logger "$direction $(echo $file | awk '{for (i=2; i<NF; i++) printf $i " "; print $NF}')" "ALWAYS" Logger "$direction $replicaPath$(echo $file | awk '{for (i=2; i<NF; i++) printf $i " "; print $NF}')" "ALWAYS"
done < "$summaryFile" done < "$summaryFile"
fi fi
} }
@ -1720,23 +1738,23 @@ function Summary {
prefix="$_LOGGER_PREFIX" prefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX="" _LOGGER_PREFIX=""
Logger "File attribute updates: INITIATOR << >> TARGET" "ALWAYS" Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>" _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" "~ >>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<" _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "~ <<"
Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS" Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID" ">>" _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID" "+ >>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID" "<<" _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID" "+ <<"
#WIP: remote deletion summary does not work #WIP: remote deletion summary does not work
Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS" Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS"
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
_SummaryFromFile "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" ">>" _SummaryFromFile "${TARGET[$__replicaDir]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" "- >>"
else else
_SummaryFromFile "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID" ">>" _SummaryFromFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID" "- >>"
fi fi
_SummaryFromFile "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "<<" _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "- <<"
_LOGGER_PREFIX="$prefix" _LOGGER_PREFIX="$prefix"
} }
@ -1883,7 +1901,6 @@ function Init {
TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix" TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
PARTIAL_DIR="${INITIATOR[$__partialDir]}" PARTIAL_DIR="${INITIATOR[$__partialDir]}"
## Set sync only function arguments for rsync ## Set sync only function arguments for rsync