WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Trying parallelism patch

This commit is contained in:
deajan 2016-07-27 13:53:31 +02:00
parent 166f67bb55
commit 4101e11f67
3 changed files with 405 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

207
dev/debug_osync.sh
View File

@ -3,8 +3,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1.1-dev PROGRAM_VERSION=1.2-dev-parallel
PROGRAM_BUILD=2016072702 PROGRAM_BUILD=2016072703
IS_STABLE=yes IS_STABLE=yes
## FUNC_BUILD=2016072702 ## FUNC_BUILD=2016072702
@ -1374,52 +1374,6 @@ function CheckCurrentConfig {
###### Osync specific functions (non shared) ###### Osync specific functions (non shared)
function _CreateStateDirsLocal {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if ! [ -d "$replica_state_dir" ]; then
$COMMAND_SUDO mkdir -p "$replica_state_dir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then
Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
fi
}
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd=
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
}
#TODO: also create deleted and backup dirs
function CreateStateDirs {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_CreateStateDirsLocal "${INITIATOR[1]}${INITIATOR[3]}"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_CreateStateDirsLocal "${TARGET[1]}${TARGET[3]}"
else
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}"
fi
}
function _CheckReplicaPathsLocal { function _CheckReplicaPathsLocal {
local replica_path="${1}" local replica_path="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -1474,6 +1428,35 @@ function _CheckReplicaPathsRemote {
fi fi
} }
#TODO: organize this function into WaitforTaskCompletion, and add optional parameter to exit on error
function WaitForPids {
local errors=0
while [ "$#" -gt 0 ]; do
for pid in "$@"; do
shift
if kill -0 "$pid" > /dev/null 2>&1; then
Logger "$pid is alive" "DEBUG"
set -- "$@" "$pid"
else
wait "$pid"
result=$?
if [ $result -eq 0 ]; then
Logger "$pid exited okay with $result" "DEBUG"
else
errors=$((errors+1))
Logger "$pid exited with bad status $result" "DEBUG"
fi
fi
done
sleep .1
done
if [ $errors -gt 0 ]; then
exit 1
fi
}
function CheckReplicaPaths { function CheckReplicaPaths {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -1487,12 +1470,67 @@ function CheckReplicaPaths {
# fi # fi
#fi #fi
_CheckReplicaPathsLocal "${INITIATOR[1]}" _CheckReplicaPathsLocal "${INITIATOR[1]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckReplicaPathsLocal "${TARGET[1]}" _CheckReplicaPathsLocal "${TARGET[1]}" &
pids="$pids $!"
else else
_CheckReplicaPathsRemote "${TARGET[1]}" _CheckReplicaPathsRemote "${TARGET[1]}" &
pids="$pids $!"
fi fi
WaitForPids $pids
}
function _CreateStateDirsLocal {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if ! [ -d "$replica_state_dir" ]; then
$COMMAND_SUDO mkdir -p "$replica_state_dir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then
Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
fi
}
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd=
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
}
#TODO: also create deleted and backup dirs
function CreateStateDirs {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_CreateStateDirsLocal "${INITIATOR[1]}${INITIATOR[3]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_CreateStateDirsLocal "${TARGET[1]}${TARGET[3]}" &
pids="$pids $!"
else
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" &
pids="$pids $!"
fi
WaitForPids $pids
} }
function _CheckDiskSpaceLocal { function _CheckDiskSpaceLocal {
@ -1539,11 +1577,14 @@ function _CheckDiskSpaceRemote {
function CheckDiskSpace { function CheckDiskSpace {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_CheckDiskSpaceLocal "${INITIATOR[1]}" _CheckDiskSpaceLocal "${INITIATOR[1]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckDiskSpaceLocal "${TARGET[1]}" _CheckDiskSpaceLocal "${TARGET[1]}" &
pids="$pids $!"
else else
_CheckDiskSpaceRemote "${TARGET[1]}" _CheckDiskSpaceRemote "${TARGET[1]}" &
pifs="$pids $!"
fi fi
} }
@ -1557,6 +1598,7 @@ function _WriteLockFilesLocal {
exit 1 exit 1
else else
Logger "Locked replica on [$lockfile]." "DEBUG" Logger "Locked replica on [$lockfile]." "DEBUG"
LOCK_LOCAL=1
fi fi
} }
@ -1578,18 +1620,23 @@ function _WriteLockFilesRemote {
exit 1 exit 1
else else
Logger "Locked remote target replica." "DEBUG" Logger "Locked remote target replica." "DEBUG"
LOCK_REMOTE_1
fi fi
} }
function WriteLockFiles { function WriteLockFiles {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_WriteLockFilesLocal "${INITIATOR[2]}" _WriteLockFilesLocal "${INITIATOR[2]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_WriteLockFilesLocal "${TARGET[2]}" _WriteLockFilesLocal "${TARGET[2]}" &
pids="$pids $!"
else else
_WriteLockFilesRemote "${TARGET[2]}" _WriteLockFilesRemote "${TARGET[2]}" &
pids="$pids $!"
fi fi
WaitForPids $pids
} }
function _CheckLocksLocal { function _CheckLocksLocal {
@ -1609,7 +1656,7 @@ function _CheckLocksLocal {
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE"
else else
Logger "There is already a local instance of osync running [$lock_pid]. Cannot start." "CRITICAL" Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -1679,6 +1726,8 @@ function CheckLocks {
exit 1 exit 1
fi fi
fi fi
#TODO: do not parallelize the detection of locks because of pids
_CheckLocksLocal "${INITIATOR[2]}" _CheckLocksLocal "${INITIATOR[2]}"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckLocksLocal "${TARGET[2]}" _CheckLocksLocal "${TARGET[2]}"
@ -1731,11 +1780,21 @@ function UnlockReplicas {
return 0 return 0
fi fi
_UnlockReplicasLocal "${INITIATOR[2]}" if [ $LOCK_LOCAL -eq 1 ]; then
_UnlockReplicasLocal "${INITIATOR[2]}" &
pids="$!"
fi
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_UnlockReplicasLocal "${TARGET[2]}" if [ $LOCK_LOCAL -eq 1 ]; then
_UnlockReplicasLocal "${TARGET[2]}" &
pids="$pids $!"
fi
else else
_UnlockReplicasRemote "${TARGET[2]}" if [ $LOCK_REMOTE -eq 1 ]; then
_UnlockReplicasRemote "${TARGET[2]}" &
pids="$pids $!"
fi
fi fi
} }
@ -2559,24 +2618,32 @@ function SoftDelete {
if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then
Logger "Running conflict backup cleanup." "NOTICE" Logger "Running conflict backup cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[4]}" $CONFLICT_BACKUP_DAYS &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids $!"
else else
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids $!"
fi fi
fi fi
WaitForPids $pids
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
Logger "Running soft deletion cleanup." "NOTICE" Logger "Running soft deletion cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[5]}" $SOFT_DELETE_DAYS _SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[5]}" $SOFT_DELETE_DAYS &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS _SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids $!"
else else
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids $!"
fi fi
fi fi
WaitForPids $pids
} }
function Init { function Init {
@ -2839,6 +2906,11 @@ PARTIAL=0
if [ "$CONFLICT_PREVALANCE" == "" ]; then if [ "$CONFLICT_PREVALANCE" == "" ]; then
CONFLICT_PREVALANCE=initiator CONFLICT_PREVALANCE=initiator
fi fi
#TODO: lock_local is in a subprocess so it won't update, rethink this
LOCK_LOCAL=1
LOCK_REMOTE=1
FORCE_UNLOCK=0 FORCE_UNLOCK=0
no_maxtime=0 no_maxtime=0
opts="" opts=""
@ -2980,6 +3052,7 @@ opts="${opts# *}"
else else
LOG_FILE="$LOGFILE" LOG_FILE="$LOGFILE"
fi fi
Logger "Script begin ! Logging to [$LOG_FILE]." "DEBUG"
if [ "$IS_STABLE" != "yes" ]; then if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build. Please use with caution." "WARN" Logger "This is an unstable dev build. Please use with caution." "WARN"

196
dev/n_osync.sh
View File

@ -3,7 +3,7 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1.1-dev PROGRAM_VERSION=1.2-dev-parallel
PROGRAM_BUILD=2016072703 PROGRAM_BUILD=2016072703
IS_STABLE=yes IS_STABLE=yes
@ -126,52 +126,6 @@ function CheckCurrentConfig {
###### Osync specific functions (non shared) ###### Osync specific functions (non shared)
function _CreateStateDirsLocal {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if ! [ -d "$replica_state_dir" ]; then
$COMMAND_SUDO mkdir -p "$replica_state_dir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then
Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
fi
}
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd=
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
}
#TODO: also create deleted and backup dirs
function CreateStateDirs {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_CreateStateDirsLocal "${INITIATOR[1]}${INITIATOR[3]}"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_CreateStateDirsLocal "${TARGET[1]}${TARGET[3]}"
else
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}"
fi
}
function _CheckReplicaPathsLocal { function _CheckReplicaPathsLocal {
local replica_path="${1}" local replica_path="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -226,6 +180,35 @@ function _CheckReplicaPathsRemote {
fi fi
} }
#TODO: organize this function into WaitforTaskCompletion, and add optional parameter to exit on error
function WaitForPids {
local errors=0
while [ "$#" -gt 0 ]; do
for pid in "$@"; do
shift
if kill -0 "$pid" > /dev/null 2>&1; then
Logger "$pid is alive" "DEBUG"
set -- "$@" "$pid"
else
wait "$pid"
result=$?
if [ $result -eq 0 ]; then
Logger "$pid exited okay with $result" "DEBUG"
else
errors=$((errors+1))
Logger "$pid exited with bad status $result" "DEBUG"
fi
fi
done
sleep .1
done
if [ $errors -gt 0 ]; then
exit 1
fi
}
function CheckReplicaPaths { function CheckReplicaPaths {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -239,12 +222,67 @@ function CheckReplicaPaths {
# fi # fi
#fi #fi
_CheckReplicaPathsLocal "${INITIATOR[1]}" _CheckReplicaPathsLocal "${INITIATOR[1]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckReplicaPathsLocal "${TARGET[1]}" _CheckReplicaPathsLocal "${TARGET[1]}" &
pids="$pids $!"
else else
_CheckReplicaPathsRemote "${TARGET[1]}" _CheckReplicaPathsRemote "${TARGET[1]}" &
pids="$pids $!"
fi fi
WaitForPids $pids
}
function _CreateStateDirsLocal {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if ! [ -d "$replica_state_dir" ]; then
$COMMAND_SUDO mkdir -p "$replica_state_dir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then
Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
fi
}
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd=
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
}
#TODO: also create deleted and backup dirs
function CreateStateDirs {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_CreateStateDirsLocal "${INITIATOR[1]}${INITIATOR[3]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_CreateStateDirsLocal "${TARGET[1]}${TARGET[3]}" &
pids="$pids $!"
else
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" &
pids="$pids $!"
fi
WaitForPids $pids
} }
function _CheckDiskSpaceLocal { function _CheckDiskSpaceLocal {
@ -291,11 +329,14 @@ function _CheckDiskSpaceRemote {
function CheckDiskSpace { function CheckDiskSpace {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_CheckDiskSpaceLocal "${INITIATOR[1]}" _CheckDiskSpaceLocal "${INITIATOR[1]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckDiskSpaceLocal "${TARGET[1]}" _CheckDiskSpaceLocal "${TARGET[1]}" &
pids="$pids $!"
else else
_CheckDiskSpaceRemote "${TARGET[1]}" _CheckDiskSpaceRemote "${TARGET[1]}" &
pifs="$pids $!"
fi fi
} }
@ -338,12 +379,16 @@ function _WriteLockFilesRemote {
function WriteLockFiles { function WriteLockFiles {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
_WriteLockFilesLocal "${INITIATOR[2]}" _WriteLockFilesLocal "${INITIATOR[2]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_WriteLockFilesLocal "${TARGET[2]}" _WriteLockFilesLocal "${TARGET[2]}" &
pids="$pids $!"
else else
_WriteLockFilesRemote "${TARGET[2]}" _WriteLockFilesRemote "${TARGET[2]}" &
pids="$pids $!"
fi fi
WaitForPids $pids
} }
function _CheckLocksLocal { function _CheckLocksLocal {
@ -363,7 +408,7 @@ function _CheckLocksLocal {
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE"
else else
Logger "There is already a local instance of osync running [$lock_pid]. Cannot start." "CRITICAL" Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -433,6 +478,8 @@ function CheckLocks {
exit 1 exit 1
fi fi
fi fi
#TODO: do not parallelize the detection of locks because of pids
_CheckLocksLocal "${INITIATOR[2]}" _CheckLocksLocal "${INITIATOR[2]}"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckLocksLocal "${TARGET[2]}" _CheckLocksLocal "${TARGET[2]}"
@ -486,16 +533,19 @@ function UnlockReplicas {
fi fi
if [ $LOCK_LOCAL -eq 1 ]; then if [ $LOCK_LOCAL -eq 1 ]; then
_UnlockReplicasLocal "${INITIATOR[2]}" _UnlockReplicasLocal "${INITIATOR[2]}" &
pids="$!"
fi fi
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
if [ $LOCK_LOCAL -eq 1 ]; then if [ $LOCK_LOCAL -eq 1 ]; then
_UnlockReplicasLocal "${TARGET[2]}" _UnlockReplicasLocal "${TARGET[2]}" &
pids="$pids $!"
fi fi
else else
if [ $LOCK_REMOTE -eq 1 ]; then if [ $LOCK_REMOTE -eq 1 ]; then
_UnlockReplicasRemote "${TARGET[2]}" _UnlockReplicasRemote "${TARGET[2]}" &
pids="$pids $!"
fi fi
fi fi
} }
@ -1320,24 +1370,32 @@ function SoftDelete {
if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then
Logger "Running conflict backup cleanup." "NOTICE" Logger "Running conflict backup cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[4]}" $CONFLICT_BACKUP_DAYS &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids $!"
else else
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids $!"
fi fi
fi fi
WaitForPids $pids
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
Logger "Running soft deletion cleanup." "NOTICE" Logger "Running soft deletion cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[5]}" $SOFT_DELETE_DAYS _SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[5]}" $SOFT_DELETE_DAYS &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS _SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids $!"
else else
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids $!"
fi fi
fi fi
WaitForPids $pids
} }
function Init { function Init {
@ -1601,8 +1659,9 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then
CONFLICT_PREVALANCE=initiator CONFLICT_PREVALANCE=initiator
fi fi
LOCK_LOCAL=0 #TODO: lock_local is in a subprocess so it won't update, rethink this
LOCK_REMOTE=0 LOCK_LOCAL=1
LOCK_REMOTE=1
FORCE_UNLOCK=0 FORCE_UNLOCK=0
no_maxtime=0 no_maxtime=0
@ -1745,6 +1804,7 @@ opts="${opts# *}"
else else
LOG_FILE="$LOGFILE" LOG_FILE="$LOGFILE"
fi fi
Logger "Script begin ! Logging to [$LOG_FILE]." "DEBUG"
if [ "$IS_STABLE" != "yes" ]; then if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build. Please use with caution." "WARN" Logger "This is an unstable dev build. Please use with caution." "WARN"

201
osync.sh
View File

@ -3,8 +3,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1.1-dev PROGRAM_VERSION=1.2-dev-parallel
PROGRAM_BUILD=2016072702 PROGRAM_BUILD=2016072703
IS_STABLE=yes IS_STABLE=yes
## FUNC_BUILD=2016072702 ## FUNC_BUILD=2016072702
@ -1298,49 +1298,6 @@ function CheckCurrentConfig {
###### Osync specific functions (non shared) ###### Osync specific functions (non shared)
function _CreateStateDirsLocal {
local replica_state_dir="${1}"
if ! [ -d "$replica_state_dir" ]; then
$COMMAND_SUDO mkdir -p "$replica_state_dir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then
Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
fi
}
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
local cmd=
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
}
#TODO: also create deleted and backup dirs
function CreateStateDirs {
_CreateStateDirsLocal "${INITIATOR[1]}${INITIATOR[3]}"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_CreateStateDirsLocal "${TARGET[1]}${TARGET[3]}"
else
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}"
fi
}
function _CheckReplicaPathsLocal { function _CheckReplicaPathsLocal {
local replica_path="${1}" local replica_path="${1}"
@ -1393,6 +1350,35 @@ function _CheckReplicaPathsRemote {
fi fi
} }
#TODO: organize this function into WaitforTaskCompletion, and add optional parameter to exit on error
function WaitForPids {
local errors=0
while [ "$#" -gt 0 ]; do
for pid in "$@"; do
shift
if kill -0 "$pid" > /dev/null 2>&1; then
Logger "$pid is alive" "DEBUG"
set -- "$@" "$pid"
else
wait "$pid"
result=$?
if [ $result -eq 0 ]; then
Logger "$pid exited okay with $result" "DEBUG"
else
errors=$((errors+1))
Logger "$pid exited with bad status $result" "DEBUG"
fi
fi
done
sleep .1
done
if [ $errors -gt 0 ]; then
exit 1
fi
}
function CheckReplicaPaths { function CheckReplicaPaths {
#INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO: investigate realpath & readlink issues on MSYS and busybox here #INITIATOR_SYNC_DIR_CANN=$(realpath "${INITIATOR[1]}") #TODO: investigate realpath & readlink issues on MSYS and busybox here
@ -1405,12 +1391,64 @@ function CheckReplicaPaths {
# fi # fi
#fi #fi
_CheckReplicaPathsLocal "${INITIATOR[1]}" _CheckReplicaPathsLocal "${INITIATOR[1]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckReplicaPathsLocal "${TARGET[1]}" _CheckReplicaPathsLocal "${TARGET[1]}" &
pids="$pids $!"
else else
_CheckReplicaPathsRemote "${TARGET[1]}" _CheckReplicaPathsRemote "${TARGET[1]}" &
pids="$pids $!"
fi fi
WaitForPids $pids
}
function _CreateStateDirsLocal {
local replica_state_dir="${1}"
if ! [ -d "$replica_state_dir" ]; then
$COMMAND_SUDO mkdir -p "$replica_state_dir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then
Logger "Cannot create state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
fi
}
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
local cmd=
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
exit 1
fi
}
#TODO: also create deleted and backup dirs
function CreateStateDirs {
_CreateStateDirsLocal "${INITIATOR[1]}${INITIATOR[3]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_CreateStateDirsLocal "${TARGET[1]}${TARGET[3]}" &
pids="$pids $!"
else
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" &
pids="$pids $!"
fi
WaitForPids $pids
} }
function _CheckDiskSpaceLocal { function _CheckDiskSpaceLocal {
@ -1454,11 +1492,14 @@ function _CheckDiskSpaceRemote {
function CheckDiskSpace { function CheckDiskSpace {
_CheckDiskSpaceLocal "${INITIATOR[1]}" _CheckDiskSpaceLocal "${INITIATOR[1]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckDiskSpaceLocal "${TARGET[1]}" _CheckDiskSpaceLocal "${TARGET[1]}" &
pids="$pids $!"
else else
_CheckDiskSpaceRemote "${TARGET[1]}" _CheckDiskSpaceRemote "${TARGET[1]}" &
pifs="$pids $!"
fi fi
} }
@ -1471,6 +1512,7 @@ function _WriteLockFilesLocal {
exit 1 exit 1
else else
Logger "Locked replica on [$lockfile]." "DEBUG" Logger "Locked replica on [$lockfile]." "DEBUG"
LOCK_LOCAL=1
fi fi
} }
@ -1491,17 +1533,22 @@ function _WriteLockFilesRemote {
exit 1 exit 1
else else
Logger "Locked remote target replica." "DEBUG" Logger "Locked remote target replica." "DEBUG"
LOCK_REMOTE_1
fi fi
} }
function WriteLockFiles { function WriteLockFiles {
_WriteLockFilesLocal "${INITIATOR[2]}" _WriteLockFilesLocal "${INITIATOR[2]}" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_WriteLockFilesLocal "${TARGET[2]}" _WriteLockFilesLocal "${TARGET[2]}" &
pids="$pids $!"
else else
_WriteLockFilesRemote "${TARGET[2]}" _WriteLockFilesRemote "${TARGET[2]}" &
pids="$pids $!"
fi fi
WaitForPids $pids
} }
function _CheckLocksLocal { function _CheckLocksLocal {
@ -1520,7 +1567,7 @@ function _CheckLocksLocal {
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE"
else else
Logger "There is already a local instance of osync running [$lock_pid]. Cannot start." "CRITICAL" Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -1588,6 +1635,8 @@ function CheckLocks {
exit 1 exit 1
fi fi
fi fi
#TODO: do not parallelize the detection of locks because of pids
_CheckLocksLocal "${INITIATOR[2]}" _CheckLocksLocal "${INITIATOR[2]}"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckLocksLocal "${TARGET[2]}" _CheckLocksLocal "${TARGET[2]}"
@ -1637,11 +1686,21 @@ function UnlockReplicas {
return 0 return 0
fi fi
_UnlockReplicasLocal "${INITIATOR[2]}" if [ $LOCK_LOCAL -eq 1 ]; then
_UnlockReplicasLocal "${INITIATOR[2]}" &
pids="$!"
fi
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_UnlockReplicasLocal "${TARGET[2]}" if [ $LOCK_LOCAL -eq 1 ]; then
_UnlockReplicasLocal "${TARGET[2]}" &
pids="$pids $!"
fi
else else
_UnlockReplicasRemote "${TARGET[2]}" if [ $LOCK_REMOTE -eq 1 ]; then
_UnlockReplicasRemote "${TARGET[2]}" &
pids="$pids $!"
fi
fi fi
} }
@ -2452,24 +2511,32 @@ function SoftDelete {
if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then
Logger "Running conflict backup cleanup." "NOTICE" Logger "Running conflict backup cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[4]}" $CONFLICT_BACKUP_DAYS &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids $!"
else else
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids $!"
fi fi
fi fi
WaitForPids $pids
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
Logger "Running soft deletion cleanup." "NOTICE" Logger "Running soft deletion cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[5]}" $SOFT_DELETE_DAYS _SoftDeleteLocal "${INITIATOR[0]}" "${INITIATOR[1]}${INITIATOR[5]}" $SOFT_DELETE_DAYS &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS _SoftDeleteLocal "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids $!"
else else
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS _SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids $!"
fi fi
fi fi
WaitForPids $pids
} }
function Init { function Init {
@ -2728,6 +2795,11 @@ PARTIAL=0
if [ "$CONFLICT_PREVALANCE" == "" ]; then if [ "$CONFLICT_PREVALANCE" == "" ]; then
CONFLICT_PREVALANCE=initiator CONFLICT_PREVALANCE=initiator
fi fi
#TODO: lock_local is in a subprocess so it won't update, rethink this
LOCK_LOCAL=1
LOCK_REMOTE=1
FORCE_UNLOCK=0 FORCE_UNLOCK=0
no_maxtime=0 no_maxtime=0
opts="" opts=""
@ -2869,6 +2941,7 @@ opts="${opts# *}"
else else
LOG_FILE="$LOGFILE" LOG_FILE="$LOGFILE"
fi fi
Logger "Script begin ! Logging to [$LOG_FILE]." "DEBUG"
if [ "$IS_STABLE" != "yes" ]; then if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build. Please use with caution." "WARN" Logger "This is an unstable dev build. Please use with caution." "WARN"