WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2016-10-21 12:38:34 +02:00
parent 9c212c1bd1
commit d844a5b696
2 changed files with 184 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
dev/debug_osync.sh
View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2 PROGRAM_VERSION=1.2-beta2
PROGRAM_BUILD=2016101702 PROGRAM_BUILD=2016102101
IS_STABLE=no IS_STABLE=no
# Execution order #__WITH_PARANOIA_DEBUG # Execution order #__WITH_PARANOIA_DEBUG
@ -45,7 +45,7 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016101901 ## FUNC_BUILD=2016102101
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -967,6 +967,21 @@ function urlDecode {
printf '%b' "${urlEncoded//%/\\x}" printf '%b' "${urlEncoded//%/\\x}"
} }
## Modified version of http://stackoverflow.com/a/8574392
## Usage: arrayContains "needle" "${haystack[@]}"
arrayContains () {
local e
if [ "$2" == "" ]; then
echo 1 && return 1
fi
for e in "${@:2}"; do
[[ "$e" == "$1" ]] && echo 0 && return 0
done
echo 1 && return 1
}
function GetLocalOS { function GetLocalOS {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -1698,6 +1713,11 @@ function CheckCurrentConfigAll {
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL" Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL"
exit 1 exit 1
fi fi
if [ "$SKIP_DELETION" != "" ] && [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ]; then
Logger "Bogus skip deletion parameter." "CRITICAL"
exit 1
fi
} }
###### Osync specific functions (non shared) ###### Osync specific functions (non shared)
@ -1920,6 +1940,11 @@ function _CheckLocksLocal {
kill -9 $lock_pid > /dev/null 2>&1 kill -9 $lock_pid > /dev/null 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE"
#rm "$lockfile"
#if [ $? != 0 ]; then
# Logger "Cannot remove lock in [$lockfile]." "CRITICAL"
# exit 1
#fi
else else
Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL" Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
@ -1991,8 +2016,7 @@ function CheckLocks {
if [ $? != 0 ]; then if [ $? != 0 ]; then
exit 1 exit 1
fi fi
fi else
_CheckLocksLocal "${INITIATOR[$__lockFile]}" & _CheckLocksLocal "${INITIATOR[$__lockFile]}" &
pids="$!" pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
@ -2008,6 +2032,7 @@ function CheckLocks {
exit 1 exit 1
fi fi
WriteLockFiles WriteLockFiles
fi
} }
function _WriteLockFilesLocal { function _WriteLockFilesLocal {
@ -2015,10 +2040,14 @@ function _WriteLockFilesLocal {
local replicaType="${2}" local replicaType="${2}"
__CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" (
set -o noclobber
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID"
)
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL"
exit 1 Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID)" "NOTICE"
return 1
else else
Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG"
fi fi
@ -2034,12 +2063,13 @@ function _WriteLockFilesRemote {
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' cmd=$SSH_CMD' "( set -o noclobber; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL" Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL"
exit 1 Loggxer "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID)" "NOTICE"
return 1
else else
Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG" Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG"
fi fi
@ -2071,9 +2101,9 @@ function WriteLockFiles {
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
pid=${pid%:*} pid=${pid%:*}
if [ $pid == $initiatorPid ]; then if [ "$pid" == "$initiatorPid" ]; then
INITIATOR_LOCK_FILE_EXISTS=false INITIATOR_LOCK_FILE_EXISTS=false
elif [ $pid == $targetPid ]; then elif [ "$pid" == "$targetPid" ]; then
TARGET_LOCK_FILE_EXISTS=false TARGET_LOCK_FILE_EXISTS=false
fi fi
done done
@ -2685,6 +2715,7 @@ function deletionPropagation {
#TODO: deletionPropagation replicaType = source replica whereas _deleteXxxxxx replicaType = dest replica #TODO: deletionPropagation replicaType = source replica whereas _deleteXxxxxx replicaType = dest replica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ]; then
replicaDir="${INITIATOR[$__replicaDir]}" replicaDir="${INITIATOR[$__replicaDir]}"
deleteDir="${INITIATOR[$__deleteDir]}" deleteDir="${INITIATOR[$__deleteDir]}"
@ -2695,6 +2726,10 @@ function deletionPropagation {
exit 1 exit 1
fi fi
else else
Logger "Skipping deletion on replica $replicaType." "NOTICE"
fi
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
if [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ]; then
replicaDir="${TARGET[$__replicaDir]}" replicaDir="${TARGET[$__replicaDir]}"
deleteDir="${TARGET[$__deleteDir]}" deleteDir="${TARGET[$__deleteDir]}"
@ -2716,6 +2751,9 @@ function deletionPropagation {
fi fi
exit 1 exit 1
fi fi
else
Logger "Skipping deletion on replica $replicaType." "NOTICE"
fi
fi fi
} }
@ -3365,6 +3403,7 @@ function Usage {
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica" echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica"
echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used" echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used"
echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets" echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets"
echo "--skip-deletion=\"\" You may skip deletion propagation on initiator or target. Valid values: initiator target initiator,target"
echo "" echo ""
echo "Additionaly, you may set most osync options at runtime. eg:" echo "Additionaly, you may set most osync options at runtime. eg:"
echo "SOFT_DELETE_DAYS=365 osync.sh --initiator=/path --target=/other/path" echo "SOFT_DELETE_DAYS=365 osync.sh --initiator=/path --target=/other/path"
@ -3501,6 +3540,11 @@ for i in "$@"; do
INSTANCE_ID=${i##*=} INSTANCE_ID=${i##*=}
opts=$opts" --instance-id=\"$INSTANCE_ID\"" opts=$opts" --instance-id=\"$INSTANCE_ID\""
;; ;;
--skip-deletion=*)
#SKIP_DELETION=${i##*=}
opts=$opts" --skip-deletion=\"${i##*=}\""
IFS=',' read -r -a SKIP_DELETION <<< ${i##*=}
;;
--on-changes) --on-changes)
sync_on_changes=true sync_on_changes=true
_NOLOCKS=true _NOLOCKS=true

64
osync.sh
View File

@ -4,14 +4,14 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2 PROGRAM_VERSION=1.2-beta2
PROGRAM_BUILD=2016101702 PROGRAM_BUILD=2016102101
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016101901 ## FUNC_BUILD=2016102101
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -901,6 +901,21 @@ function urlDecode {
printf '%b' "${urlEncoded//%/\\x}" printf '%b' "${urlEncoded//%/\\x}"
} }
## Modified version of http://stackoverflow.com/a/8574392
## Usage: arrayContains "needle" "${haystack[@]}"
arrayContains () {
local e
if [ "$2" == "" ]; then
echo 1 && return 1
fi
for e in "${@:2}"; do
[[ "$e" == "$1" ]] && echo 0 && return 0
done
echo 1 && return 1
}
function GetLocalOS { function GetLocalOS {
local localOsVar local localOsVar
@ -1565,6 +1580,11 @@ function CheckCurrentConfigAll {
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL" Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL"
exit 1 exit 1
fi fi
if [ "$SKIP_DELETION" != "" ] && [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ]; then
Logger "Bogus skip deletion parameter." "CRITICAL"
exit 1
fi
} }
###### Osync specific functions (non shared) ###### Osync specific functions (non shared)
@ -1777,6 +1797,11 @@ function _CheckLocksLocal {
kill -9 $lock_pid > /dev/null 2>&1 kill -9 $lock_pid > /dev/null 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE"
#rm "$lockfile"
#if [ $? != 0 ]; then
# Logger "Cannot remove lock in [$lockfile]." "CRITICAL"
# exit 1
#fi
else else
Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL" Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
@ -1846,8 +1871,7 @@ function CheckLocks {
if [ $? != 0 ]; then if [ $? != 0 ]; then
exit 1 exit 1
fi fi
fi else
_CheckLocksLocal "${INITIATOR[$__lockFile]}" & _CheckLocksLocal "${INITIATOR[$__lockFile]}" &
pids="$!" pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
@ -1863,16 +1887,21 @@ function CheckLocks {
exit 1 exit 1
fi fi
WriteLockFiles WriteLockFiles
fi
} }
function _WriteLockFilesLocal { function _WriteLockFilesLocal {
local lockfile="${1}" local lockfile="${1}"
local replicaType="${2}" local replicaType="${2}"
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" (
set -o noclobber
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID"
)
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL"
exit 1 Logger "Command output\n$($RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID)" "NOTICE"
return 1
else else
Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG" Logger "Locked local $replicaType replica in [$lockfile]." "DEBUG"
fi fi
@ -1887,12 +1916,13 @@ function _WriteLockFilesRemote {
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1' cmd=$SSH_CMD' "( set -o noclobber; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL" Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL"
exit 1 Loggxer "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID)" "NOTICE"
return 1
else else
Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG" Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG"
fi fi
@ -1923,9 +1953,9 @@ function WriteLockFiles {
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
pid=${pid%:*} pid=${pid%:*}
if [ $pid == $initiatorPid ]; then if [ "$pid" == "$initiatorPid" ]; then
INITIATOR_LOCK_FILE_EXISTS=false INITIATOR_LOCK_FILE_EXISTS=false
elif [ $pid == $targetPid ]; then elif [ "$pid" == "$targetPid" ]; then
TARGET_LOCK_FILE_EXISTS=false TARGET_LOCK_FILE_EXISTS=false
fi fi
done done
@ -2525,6 +2555,7 @@ function deletionPropagation {
#TODO: deletionPropagation replicaType = source replica whereas _deleteXxxxxx replicaType = dest replica #TODO: deletionPropagation replicaType = source replica whereas _deleteXxxxxx replicaType = dest replica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ]; then
replicaDir="${INITIATOR[$__replicaDir]}" replicaDir="${INITIATOR[$__replicaDir]}"
deleteDir="${INITIATOR[$__deleteDir]}" deleteDir="${INITIATOR[$__deleteDir]}"
@ -2535,6 +2566,10 @@ function deletionPropagation {
exit 1 exit 1
fi fi
else else
Logger "Skipping deletion on replica $replicaType." "NOTICE"
fi
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
if [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -ne 0 ]; then
replicaDir="${TARGET[$__replicaDir]}" replicaDir="${TARGET[$__replicaDir]}"
deleteDir="${TARGET[$__deleteDir]}" deleteDir="${TARGET[$__deleteDir]}"
@ -2556,6 +2591,9 @@ function deletionPropagation {
fi fi
exit 1 exit 1
fi fi
else
Logger "Skipping deletion on replica $replicaType." "NOTICE"
fi
fi fi
} }
@ -3198,6 +3236,7 @@ function Usage {
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica" echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica"
echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used" echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used"
echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets" echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets"
echo "--skip-deletion=\"\" You may skip deletion propagation on initiator or target. Valid values: initiator target initiator,target"
echo "" echo ""
echo "Additionaly, you may set most osync options at runtime. eg:" echo "Additionaly, you may set most osync options at runtime. eg:"
echo "SOFT_DELETE_DAYS=365 osync.sh --initiator=/path --target=/other/path" echo "SOFT_DELETE_DAYS=365 osync.sh --initiator=/path --target=/other/path"
@ -3333,6 +3372,11 @@ for i in "$@"; do
INSTANCE_ID=${i##*=} INSTANCE_ID=${i##*=}
opts=$opts" --instance-id=\"$INSTANCE_ID\"" opts=$opts" --instance-id=\"$INSTANCE_ID\""
;; ;;
--skip-deletion=*)
#SKIP_DELETION=${i##*=}
opts=$opts" --skip-deletion=\"${i##*=}\""
IFS=',' read -r -a SKIP_DELETION <<< ${i##*=}
;;
--on-changes) --on-changes)
sync_on_changes=true sync_on_changes=true
_NOLOCKS=true _NOLOCKS=true