WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2016-11-18 09:05:06 +01:00
parent 7283729eae
commit cafbf43222
3 changed files with 415 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

343
dev/debug_osync.sh
View File

@ -3,8 +3,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2 PROGRAM_VERSION=1.2-beta3
PROGRAM_BUILD=2016111502 PROGRAM_BUILD=2016111704
IS_STABLE=no IS_STABLE=no
# Execution order #__WITH_PARANOIA_DEBUG # Execution order #__WITH_PARANOIA_DEBUG
@ -45,17 +45,22 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111503 ## FUNC_BUILD=2016111705
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
## PROGRAM=program-name ## PROGRAM=program-name
## INSTANCE_ID=program-instance-name ## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no ## _DEBUG=yes/no
## _LOGGER_STDERR=true/false ## _LOGGER_LOGGER_SILENT=true/false
## _LOGGER_LOGGER_VERBOSE=true/false
## _LOGGER_ERR_ONLY=true/false ## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/"" ## _LOGGER_PREFIX="date"/"time"/""
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile
#TODO: Windows checks, check sendmail & mailsend #TODO: Windows checks, check sendmail & mailsend
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
@ -71,10 +76,10 @@ MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warni
# Environment variables that can be overriden by programs # Environment variables that can be overriden by programs
_DRYRUN=false _DRYRUN=false
_SILENT=false _LOGGER_SILENT=false
_VERBOSE=false _LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date" _LOGGER_PREFIX="date"
_LOGGER_STDERR=false
if [ "$KEEP_LOGGING" == "" ]; then if [ "$KEEP_LOGGING" == "" ]; then
KEEP_LOGGING=1801 KEEP_LOGGING=1801
fi fi
@ -83,9 +88,6 @@ fi
ERROR_ALERT=false ERROR_ALERT=false
WARN_ALERT=false WARN_ALERT=false
# Log from current run
CURRENT_LOG=""
## allow function call checks #__WITH_PARANOIA_DEBUG ## allow function call checks #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG
_DEBUG=yes #__WITH_PARANOIA_DEBUG _DEBUG=yes #__WITH_PARANOIA_DEBUG
@ -95,13 +97,13 @@ fi #__WITH_PARANOIA_DEBUG
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=1 SLEEP_TIME=.05
fi fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_VERBOSE=true _LOGGER_VERBOSE=true
fi fi
SCRIPT_PID=$$ SCRIPT_PID=$$
@ -148,28 +150,43 @@ function Dummy {
# Sub function of Logger # Sub function of Logger
function _Logger { function _Logger {
local svalue="${1}" # What to log to stdout local logValue="${1}" # Log to file
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value local stdValue="${2}" # Log to screeen
local evalue="${3}" # What to log to stderr local toStderr="${3:-false}" # Log to stderr instead of stdout
echo -e "$lvalue" >> "$LOG_FILE" echo -e "$logValue" >> "$LOG_FILE"
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" # Current log file
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
cat <<< "$evalue" 1>&2 if [ $toStderr == true ]; then
elif [ "$_SILENT" == false ]; then # Force stderr color in subshell
echo -e "$svalue" (>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi fi
} }
# General log function with log levels: # General log function with log levels:
# CRITICAL, ERROR, WARN are colored in stdout, prefixed in stderr
# NOTICE is standard level # Environment variables
# VERBOSE is only sent to stdout / stderr if _VERBOSE=true # _LOGGER_SILENT: Disables any output to stdout & stderr
# DEBUG & PARANOIA_DEBUG are only sent if _DEBUG=yes # _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
function Logger { function Logger {
local value="${1}" # Sentence to log (in double quotes) local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, VERBOSE, NOTICE, WARN, ERROR, CRITIAL local level="${2}" # Log level
if [ "$_LOGGER_PREFIX" == "time" ]; then if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - " prefix="TIME: $SECONDS - "
@ -180,42 +197,44 @@ function Logger {
fi fi
if [ "$level" == "CRITICAL" ]; then if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix($level):$value" "$prefix\e[41m$value\e[0m" true
ERROR_ALERT=true ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true ERROR_ALERT=true
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "WARN" ]; then elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[33m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true WARN_ALERT=true
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID"
return return
elif [ "$level" == "NOTICE" ]; then elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" _Logger "$prefix$value" "$prefix$value"
fi fi
return return
elif [ "$level" == "VERBOSE" ]; then elif [ "$level" == "VERBOSE" ]; then
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix$value" _Logger "$prefix:$value" "$prefix$value"
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
if [ $_SILENT != true ]; then _Logger "$prefix$value" "$prefix$value"
_Logger "$prefix$value" "$prefix$level:$value" "$level:$value"
fi
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" _Logger "$prefix$value" "$prefix$value"
return return
fi fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "$prefix$value" #__WITH_PARANOIA_DEBUG _Logger "$prefix$value" "$prefix$value" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG
else else
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m"
_Logger "Value was: $prefix$value" _Logger "Value was: $prefix$value"
@ -242,7 +261,7 @@ function QuickLogger {
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ $_SILENT == true ]; then if [ $_LOGGER_SILENT == true ]; then
_QuickLogger "$value" "log" _QuickLogger "$value" "log"
else else
_QuickLogger "$value" "stdout" _QuickLogger "$value" "stdout"
@ -305,10 +324,10 @@ function SendAlert {
__CheckArguments 0-1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0-1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local attachment= local attachment
local attachmentFile= local attachmentFile
local subject= local subject
local body= local body
if [ "$DESTINATION_MAILS" == "" ]; then if [ "$DESTINATION_MAILS" == "" ]; then
return 0 return 0
@ -333,7 +352,9 @@ function SendAlert {
else else
attachment=true attachment=true
fi fi
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG" if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)"
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -521,7 +542,7 @@ function TrapError {
local line="$1" local line="$1"
local code="${2:-1}" local code="${2:-1}"
if [ $_SILENT == false ]; then if [ $_LOGGER_SILENT == false ]; then
echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m"
fi fi
} }
@ -548,7 +569,7 @@ function LoadConfigFile {
} }
function Spinner { function Spinner {
if [ $_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0 return 0
fi fi
@ -1091,7 +1112,7 @@ function RunLocalCommand {
Logger "Command failed." "ERROR" Logger "Command failed." "ERROR"
fi fi
if [ $_VERBOSE == true ] || [ $retval -ne 0 ]; then if [ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]; then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
@ -1126,7 +1147,7 @@ function RunRemoteCommand {
Logger "Command failed." "ERROR" Logger "Command failed." "ERROR"
fi fi
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE == true ] || [ $retval -ne 0 ]) if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ])
then then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
@ -1622,12 +1643,19 @@ function TrapStop {
function TrapQuit { function TrapQuit {
local exitcode local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=true
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=true
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
UnlockReplicas UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook RunAfterHook
fi fi
CleanUp
Logger "$PROGRAM finished with errors." "ERROR" Logger "$PROGRAM finished with errors." "ERROR"
if [ "$_DEBUG" != "yes" ] if [ "$_DEBUG" != "yes" ]
then then
@ -1641,7 +1669,6 @@ function TrapQuit {
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook RunAfterHook
fi fi
CleanUp
Logger "$PROGRAM finished with warnings." "WARN" Logger "$PROGRAM finished with warnings." "WARN"
if [ "$_DEBUG" != "yes" ] if [ "$_DEBUG" != "yes" ]
then then
@ -1653,11 +1680,11 @@ function TrapQuit {
else else
UnlockReplicas UnlockReplicas
RunAfterHook RunAfterHook
CleanUp
Logger "$PROGRAM finished." "NOTICE" Logger "$PROGRAM finished." "NOTICE"
exitcode=0 exitcode=0
fi fi
CleanUp
KillChilds $$ > /dev/null 2>&1 KillChilds $$ > /dev/null 2>&1
exit $exitcode exit $exitcode
@ -1705,6 +1732,8 @@ function CheckCurrentConfig {
function CheckCurrentConfigAll { function CheckCurrentConfigAll {
__CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local tmp
if [ "$INSTANCE_ID" == "" ]; then if [ "$INSTANCE_ID" == "" ]; then
Logger "No INSTANCE_ID defined in config file." "CRITICAL" Logger "No INSTANCE_ID defined in config file." "CRITICAL"
exit 1 exit 1
@ -1726,9 +1755,13 @@ function CheckCurrentConfigAll {
exit 1 exit 1
fi fi
if [ "$SKIP_DELETION" != "" ] && [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then if [ "$SKIP_DELETION" != "" ]; then
Logger "Bogus skip deletion parameter." "CRITICAL" tmp="$SKIP_DELETION"
exit 1 IFS=',' read -r -a SKIP_DELETION <<< "$tmp"
if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then
Logger "Bogus skip deletion parameter [$SKIP_DELETION]." "CRITICAL"
exit 1
fi
fi fi
} }
@ -1950,27 +1983,41 @@ function CreateStateDirs {
function _CheckLocksLocal { function _CheckLocksLocal {
local lockfile="${1}" local lockfile="${1}"
__CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG local replicaType="${2}"
local lockfile_content __CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local lock_pid
local lock_instance_id
if [ -f "$lockfile" ]; then local lockfileContent
lockfile_content=$(cat $lockfile) local lockPid
Logger "Master lock pid present: $lockfile_content" "DEBUG" local lockInstanceID
lock_pid=${lockfile_content%@*}
lock_instance_id=${lockfile_content#*@} if [ -s "$lockfile" ]; then
kill -9 $lock_pid > /dev/null 2>&1 lockfileContent=$(cat $lockfile)
Logger "Master lock pid present: $lockfileContent" "DEBUG"
lockPid=${lockfileContent%@*}
if [ $(IsInteger $lockPid) -ne 1 ]; then
Logger "Invalid pid [$lockPid] in local replica." "CRITICAL"
exit 1
fi
lockInstanceID=${lockfileContent#*@}
if [ "$lockInstanceID" == "" ]; then
Logger "Invalid instance id [$lockInstanceID] in local replica." "CRITICAL"
exit 1
Logger "Local $replicaType lock is: [$lockPid@$lockInstanceID]." "DEBUG"
fi
kill -0 $lockPid > /dev/null 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a local dead osync lock [$lockPid@$lockInstanceID] that is no longer running. Resuming." "NOTICE"
#rm "$lockfile" if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
#if [ $? != 0 ]; then # REPLICA_OVERWRITE_LOCK disables noclobber option in WriteLock functions
# Logger "Cannot remove lock in [$lockfile]." "CRITICAL" INITIATOR_OVERWRITE_LOCK=true
# exit 1 elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
#fi TARGET_OVERWRITE_LOCK=true
fi
else else
Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL" Logger "There is already a local instance [$lockPid@$lockInstanceID] of osync running for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -1981,9 +2028,9 @@ function _CheckLocksRemote {
__CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local cmd local cmd
local lock_pid local lockPid
local lock_instance_id local lockInstanceID
local lockfile_content local lockfileContent
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
@ -1992,34 +2039,42 @@ function _CheckLocksRemote {
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
if [ $? != 0 ]; then if [ $? != 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "Cannot check remote replica lock." "CRITICAL"
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID) exit 1
else
Logger "Cannot get remote lockfile." "CRITICAL"
exit 1
fi
fi fi
lock_pid=${lockfile_content%@*} if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
lock_instance_id=${lockfile_content#*@} lockfileContent="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)"
if [ "$lock_pid" != "" ] && [ "$lock_instance_id" != "" ]; then lockPid=${lockfileContent%@*}
Logger "Remote lock is: $lock_pid@$lock_instance_id" "DEBUG" if [ $(IsInteger $lockPid) -ne 1 ]; then
Logger "Invalid pid [$lockPid] in remote replica lock." "CRITICAL"
exit 1
fi
lockInstanceID=${lockfileContent#*@}
if [ "$lockInstanceID" == "" ]; then
Logger "Invalid instance id [$lockInstanceID] in remote replica." "CRITICAL"
exit 1
fi
kill -0 $lock_pid > /dev/null 2>&1 Logger "Remote lock is: [$lockPid@$lockInstanceID]" "DEBUG"
kill -0 $lockPid > /dev/null 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
if [ "$lock_instance_id" == "$INSTANCE_ID" ]; then if [ "$lockInstanceID" == "$INSTANCE_ID" ]; then
Logger "There is a dead osync lock on target replica that corresponds to this initiator sync id [$lock_instance_id]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a remote dead osync lock [$lockPid@lockInstanceID] on target replica that corresponds to this initiator INSTANCE_ID. Pid [$lockPid] no longer running. Resuming." "NOTICE"
TARGET_OVERWRITE_LOCK=true
else else
if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]; then if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]; then
Logger "WARNING: There is a dead osync lock on target replica that does not correspond to this initiator sync-id [$lock_instance_id]. Forcing resume." "WARN" Logger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Forcing resume." "WARN"
TARGET_OVERWRITE_LOCK=true
else else
Logger "There is a dead osync lock on target replica that does not correspond to this initiator sync-id [$lock_instance_id]. Will not resume." "CRITICAL" Logger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Will not resume." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
else else
Logger "There is already a local instance of osync that locks target replica [$lock_pid@$lock_instance_id]. Cannot start." "CRITICAL" Logger "There is already a local instance of osync that locks target replica [$lockPid@$lockInstanceID]. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -2041,10 +2096,10 @@ function CheckLocks {
exit 1 exit 1
fi fi
else else
_CheckLocksLocal "${INITIATOR[$__lockFile]}" & _CheckLocksLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" &
pids="$!" pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckLocksLocal "${TARGET[$__lockFile]}" & _CheckLocksLocal "${TARGET[$__lockFile]}" "${INITIATOR[$__type]}" &
pids="$pids;$!" pids="$pids;$!"
else else
_CheckLocksRemote "${TARGET[$__lockFile]}" & _CheckLocksRemote "${TARGET[$__lockFile]}" &
@ -2062,10 +2117,14 @@ function CheckLocks {
function _WriteLockFilesLocal { function _WriteLockFilesLocal {
local lockfile="${1}" local lockfile="${1}"
local replicaType="${2}" local replicaType="${2}"
__CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG local overwrite="${3:-false}"
__CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
( (
set -o noclobber if [ $overwrite == true ]; then
set -o noclobber
fi
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID" $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID"
) )
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -2080,19 +2139,23 @@ function _WriteLockFilesLocal {
function _WriteLockFilesRemote { function _WriteLockFilesRemote {
local lockfile="${1}" local lockfile="${1}"
local replicaType="${2}" local replicaType="${2}"
__CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG local overwrite="${3-false}"
__CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local cmd local cmd
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
cmd=$SSH_CMD' "( set -o noclobber; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID' cmd=$SSH_CMD' "( if [ $overwrite == true ]; then set -o noclobber; fi; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID'
#WIP
#cmd=$SSH_CMD' "( set -o noclobber; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL" Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL"
Loggxer "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID)" "NOTICE"
return 1 return 1
else else
Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG" Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG"
@ -2107,14 +2170,14 @@ function WriteLockFiles {
local pidArray local pidArray
local pid local pid
_WriteLockFilesLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}"& _WriteLockFilesLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" $INITIATOR_LOCK_OVERWRITE &
initiatorPid="$!" initiatorPid="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_WriteLockFilesLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & _WriteLockFilesLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $TARGET_LOCK_OVERWRITE &
targetPid="$!" targetPid="$!"
else else
_WriteLockFilesRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & _WriteLockFilesRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $TARGET_LOCK_OVERWRITE &
targetPid="$!" targetPid="$!"
fi fi
@ -2212,11 +2275,11 @@ function treeList {
local replicaType="${2}" # replica type: initiator, target local replicaType="${2}" # replica type: initiator, target
local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType)
__CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local escapedReplicaPath local escapedReplicaPath
local rsyncCmd local rsyncCmd
__CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
escapedReplicaPath=$(EscapeSpaces "$replicaPath") escapedReplicaPath=$(EscapeSpaces "$replicaPath")
Logger "Creating $replicaType replica file list [$replicaPath]." "NOTICE" Logger "Creating $replicaType replica file list [$replicaPath]." "NOTICE"
@ -2255,6 +2318,17 @@ function deleteList {
local cmd local cmd
local failedDeletionListFromReplica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
failedDeletionListFromReplica="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
failedDeletionListFromReplica="${INITIATOR[$__type]}"
else
Logger "Bogus replicaType in [${FUNCNAME[0]}]." "CRITICAL"
exit 1
fi
Logger "Creating $replicaType replica deleted file list." "NOTICE" Logger "Creating $replicaType replica deleted file list." "NOTICE"
if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeAfterFileNoSuffix]}" ]; then if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeAfterFileNoSuffix]}" ]; then
## Same functionnality, comm is much faster than grep but is not available on every platform ## Same functionnality, comm is much faster than grep but is not available on every platform
@ -2270,10 +2344,10 @@ function deleteList {
retval=$? retval=$?
# Add delete failed file list to current delete list and then empty it # Add delete failed file list to current delete list and then empty it
if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" ]; then if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$failedDeletionListFromReplica${INITIATOR[$__failedDeletedListFile]}" ]; then
cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$failedDeletionListFromReplica${INITIATOR[$__failedDeletedListFile]}" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
if [ $? == 0 ]; then if [ $? == 0 ]; then
rm -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" rm -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$failedDeletionListFromReplica${INITIATOR[$__failedDeletedListFile]}"
else else
Logger "Cannot add failed deleted list to current deleted list for replica [$replicaType]." "ERROR" Logger "Cannot add failed deleted list to current deleted list for replica [$replicaType]." "ERROR"
fi fi
@ -2635,7 +2709,7 @@ function _deleteRemote {
exit 1 exit 1
fi fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETE_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _LOGGER_VERBOSE=$_LOGGER_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETE_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1
## The following lines are executed remotely ## The following lines are executed remotely
function _logger { function _logger {
@ -2662,7 +2736,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
_logger "$prefix$value" _logger "$prefix$value"
return return
elif [ "$level" == "VERBOSE" ]; then elif [ "$level" == "VERBOSE" ]; then
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
_logger "$prefix$value" _logger "$prefix$value"
fi fi
return return
@ -2706,12 +2780,12 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
# In order to keep full path on soft deletion, create parent directories before move # In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")" parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then if [ "$parentdir" != "." ]; then
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE"
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE"
else else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
fi fi
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR" Logger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR"
@ -2725,10 +2799,9 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
else else
if [ $_DRYRUN == false ]; then if [ $_DRYRUN == false ]; then
if [ -e "$REPLICA_DIR$files" ]; then if [ -e "$REPLICA_DIR$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
$result=$?
Logger "Deleting [$REPLICA_DIR$files]." "VERBOSE" Logger "Deleting [$REPLICA_DIR$files]." "VERBOSE"
if [ $result != 0 ]; then $COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then
Logger "Cannot delete [$REPLICA_DIR$files]." "ERROR" Logger "Cannot delete [$REPLICA_DIR$files]." "ERROR"
echo "$files" >> "$FAILED_DELETE_LIST" echo "$files" >> "$FAILED_DELETE_LIST"
else else
@ -2977,8 +3050,7 @@ function Sync {
## Step 2 ## Step 2
if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
#TODO(high) only -X and -A (xattr & acl) should trigger this if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
if [ "$RSYNC_ATTR_ARGS" != "" ]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -3152,7 +3224,7 @@ function _SoftDeleteLocal {
Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE"
fi fi
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
$COMMAND_SUDO $FIND_CMD "$replicaDeletionPath/" -type f -ctime +$changeTime -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath/" -type f -ctime +$changeTime -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE"
@ -3203,7 +3275,7 @@ function _SoftDeleteRemote {
Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE"
fi fi
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replicaDeletionPath'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type f -ctime +'$changeTime' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type d -empty -ctime '$changeTime' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replicaDeletionPath'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type f -ctime +'$changeTime' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type d -empty -ctime '$changeTime' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
@ -3269,7 +3341,7 @@ function _SummaryFromFile {
local summaryFile="${2}" local summaryFile="${2}"
local direction="${3}" local direction="${3}"
__CheckArguments 2 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
if [ -f "$summaryFile" ]; then if [ -f "$summaryFile" ]; then
while read -r file; do while read -r file; do
@ -3454,7 +3526,7 @@ function Init {
## Set sync only function arguments for rsync ## Set sync only function arguments for rsync
SYNC_OPTS="-u" SYNC_OPTS="-u"
if [ $_VERBOSE == true ] || [ $_SUMMARY == true ]; then if [ $_LOGGER_VERBOSE == true ] || [ $_SUMMARY == true ]; then
SYNC_OPTS=$SYNC_OPTS" -i" SYNC_OPTS=$SYNC_OPTS" -i"
fi fi
@ -3467,8 +3539,8 @@ function Init {
## Conflict options ## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__backupDir]}\"" INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[$__backupDir]}\""
TARGET_BACKUP="--backup --backup-dir=\"${TARGET[$__replicaDir]}${TARGET[$__backupDir]}\"" TARGET_BACKUP="--backup --backup-dir=\"${TARGET[$__backupDir]}\""
if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then
INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
@ -3619,11 +3691,11 @@ for i in "$@"; do
opts=$opts" --dry" opts=$opts" --dry"
;; ;;
--silent) --silent)
_SILENT=true _LOGGER_SILENT=true
opts=$opts" --silent" opts=$opts" --silent"
;; ;;
--verbose) --verbose)
_VERBOSE=true _LOGGER_VERBOSE=true
opts=$opts" --verbose" opts=$opts" --verbose"
;; ;;
--stats) --stats)
@ -3671,20 +3743,17 @@ for i in "$@"; do
;; ;;
--skip-deletion=*) --skip-deletion=*)
opts=$opts" --skip-deletion=\"${i##*=}\"" opts=$opts" --skip-deletion=\"${i##*=}\""
IFS=',' read -r -a SKIP_DELETION <<< ${i##*=} SKIP_DELETION=${##*=}
;; ;;
--on-changes) --on-changes)
sync_on_changes=true sync_on_changes=true
_NOLOCKS=true _NOLOCKS=true
_LOGGER_PREFIX="date" _LOGGER_PREFIX="date"
_LOGGER_STDERR=true
;; ;;
--no-locks) --no-locks)
_NOLOCKS=true _NOLOCKS=true
;; ;;
--errors-only) --errors-only)
#TODO: let err_only only output to stderr
#_LOGGER_STDERR=true
_LOGGER_ERR_ONLY=true _LOGGER_ERR_ONLY=true
;; ;;
--summary) --summary)
@ -3739,6 +3808,18 @@ opts="${opts# *}"
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
if [ "$PRESERVE_ACL" == "" ]; then
PRESERVE_ACL="yes"
fi
if [ "$PRESERVE_XATTR" == "" ]; then
PRESERVE_XATTR="yes"
fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi
MIN_WAIT=30 MIN_WAIT=30
else else
ConfigFile="${1}" ConfigFile="${1}"

2
install.sh
View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
PROGRAM=osync PROGRAM=osync
PROGRAM_VERSION=1.2-beta2 PROGRAM_VERSION=1.2-beta3
PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016111201 SCRIPT_BUILD=2016111201

323
osync.sh
View File

@ -3,25 +3,30 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2 PROGRAM_VERSION=1.2-beta3
PROGRAM_BUILD=2016111502 PROGRAM_BUILD=2016111704
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111503 ## FUNC_BUILD=2016111705
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
## PROGRAM=program-name ## PROGRAM=program-name
## INSTANCE_ID=program-instance-name ## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no ## _DEBUG=yes/no
## _LOGGER_STDERR=true/false ## _LOGGER_LOGGER_SILENT=true/false
## _LOGGER_LOGGER_VERBOSE=true/false
## _LOGGER_ERR_ONLY=true/false ## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/"" ## _LOGGER_PREFIX="date"/"time"/""
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile
#TODO: Windows checks, check sendmail & mailsend #TODO: Windows checks, check sendmail & mailsend
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
@ -37,10 +42,10 @@ MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warni
# Environment variables that can be overriden by programs # Environment variables that can be overriden by programs
_DRYRUN=false _DRYRUN=false
_SILENT=false _LOGGER_SILENT=false
_VERBOSE=false _LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date" _LOGGER_PREFIX="date"
_LOGGER_STDERR=false
if [ "$KEEP_LOGGING" == "" ]; then if [ "$KEEP_LOGGING" == "" ]; then
KEEP_LOGGING=1801 KEEP_LOGGING=1801
fi fi
@ -49,21 +54,18 @@ fi
ERROR_ALERT=false ERROR_ALERT=false
WARN_ALERT=false WARN_ALERT=false
# Log from current run
CURRENT_LOG=""
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=1 SLEEP_TIME=.05
fi fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_VERBOSE=true _LOGGER_VERBOSE=true
fi fi
SCRIPT_PID=$$ SCRIPT_PID=$$
@ -109,28 +111,43 @@ function Dummy {
# Sub function of Logger # Sub function of Logger
function _Logger { function _Logger {
local svalue="${1}" # What to log to stdout local logValue="${1}" # Log to file
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value local stdValue="${2}" # Log to screeen
local evalue="${3}" # What to log to stderr local toStderr="${3:-false}" # Log to stderr instead of stdout
echo -e "$lvalue" >> "$LOG_FILE" echo -e "$logValue" >> "$LOG_FILE"
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" # Current log file
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
cat <<< "$evalue" 1>&2 if [ $toStderr == true ]; then
elif [ "$_SILENT" == false ]; then # Force stderr color in subshell
echo -e "$svalue" (>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi fi
} }
# General log function with log levels: # General log function with log levels:
# CRITICAL, ERROR, WARN are colored in stdout, prefixed in stderr
# NOTICE is standard level # Environment variables
# VERBOSE is only sent to stdout / stderr if _VERBOSE=true # _LOGGER_SILENT: Disables any output to stdout & stderr
# DEBUG & PARANOIA_DEBUG are only sent if _DEBUG=yes # _LOGGER_STD_ERR: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
function Logger { function Logger {
local value="${1}" # Sentence to log (in double quotes) local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, VERBOSE, NOTICE, WARN, ERROR, CRITIAL local level="${2}" # Log level
if [ "$_LOGGER_PREFIX" == "time" ]; then if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - " prefix="TIME: $SECONDS - "
@ -141,35 +158,37 @@ function Logger {
fi fi
if [ "$level" == "CRITICAL" ]; then if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix($level):$value" "$prefix\e[41m$value\e[0m" true
ERROR_ALERT=true ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true ERROR_ALERT=true
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "WARN" ]; then elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[33m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true WARN_ALERT=true
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID"
return return
elif [ "$level" == "NOTICE" ]; then elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" _Logger "$prefix$value" "$prefix$value"
fi fi
return return
elif [ "$level" == "VERBOSE" ]; then elif [ "$level" == "VERBOSE" ]; then
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix$value" _Logger "$prefix:$value" "$prefix$value"
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
if [ $_SILENT != true ]; then _Logger "$prefix$value" "$prefix$value"
_Logger "$prefix$value" "$prefix$level:$value" "$level:$value"
fi
return return
elif [ "$level" == "DEBUG" ]; then elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" _Logger "$prefix$value" "$prefix$value"
return return
fi fi
else else
@ -196,7 +215,7 @@ function QuickLogger {
local value="${1}" local value="${1}"
if [ $_SILENT == true ]; then if [ $_LOGGER_SILENT == true ]; then
_QuickLogger "$value" "log" _QuickLogger "$value" "log"
else else
_QuickLogger "$value" "stdout" _QuickLogger "$value" "stdout"
@ -256,10 +275,10 @@ function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
local attachment= local attachment
local attachmentFile= local attachmentFile
local subject= local subject
local body= local body
if [ "$DESTINATION_MAILS" == "" ]; then if [ "$DESTINATION_MAILS" == "" ]; then
return 0 return 0
@ -284,7 +303,9 @@ function SendAlert {
else else
attachment=true attachment=true
fi fi
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG" if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)"
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -471,7 +492,7 @@ function TrapError {
local line="$1" local line="$1"
local code="${2:-1}" local code="${2:-1}"
if [ $_SILENT == false ]; then if [ $_LOGGER_SILENT == false ]; then
echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m" echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m"
fi fi
} }
@ -497,7 +518,7 @@ function LoadConfigFile {
} }
function Spinner { function Spinner {
if [ $_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0 return 0
fi fi
@ -1022,7 +1043,7 @@ function RunLocalCommand {
Logger "Command failed." "ERROR" Logger "Command failed." "ERROR"
fi fi
if [ $_VERBOSE == true ] || [ $retval -ne 0 ]; then if [ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]; then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
@ -1056,7 +1077,7 @@ function RunRemoteCommand {
Logger "Command failed." "ERROR" Logger "Command failed." "ERROR"
fi fi
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE == true ] || [ $retval -ne 0 ]) if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ])
then then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
@ -1492,12 +1513,19 @@ function TrapStop {
function TrapQuit { function TrapQuit {
local exitcode local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=true
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=true
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
UnlockReplicas UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook RunAfterHook
fi fi
CleanUp
Logger "$PROGRAM finished with errors." "ERROR" Logger "$PROGRAM finished with errors." "ERROR"
if [ "$_DEBUG" != "yes" ] if [ "$_DEBUG" != "yes" ]
then then
@ -1511,7 +1539,6 @@ function TrapQuit {
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook RunAfterHook
fi fi
CleanUp
Logger "$PROGRAM finished with warnings." "WARN" Logger "$PROGRAM finished with warnings." "WARN"
if [ "$_DEBUG" != "yes" ] if [ "$_DEBUG" != "yes" ]
then then
@ -1523,11 +1550,11 @@ function TrapQuit {
else else
UnlockReplicas UnlockReplicas
RunAfterHook RunAfterHook
CleanUp
Logger "$PROGRAM finished." "NOTICE" Logger "$PROGRAM finished." "NOTICE"
exitcode=0 exitcode=0
fi fi
CleanUp
KillChilds $$ > /dev/null 2>&1 KillChilds $$ > /dev/null 2>&1
exit $exitcode exit $exitcode
@ -1572,6 +1599,8 @@ function CheckCurrentConfig {
function CheckCurrentConfigAll { function CheckCurrentConfigAll {
local tmp
if [ "$INSTANCE_ID" == "" ]; then if [ "$INSTANCE_ID" == "" ]; then
Logger "No INSTANCE_ID defined in config file." "CRITICAL" Logger "No INSTANCE_ID defined in config file." "CRITICAL"
exit 1 exit 1
@ -1593,9 +1622,13 @@ function CheckCurrentConfigAll {
exit 1 exit 1
fi fi
if [ "$SKIP_DELETION" != "" ] && [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then if [ "$SKIP_DELETION" != "" ]; then
Logger "Bogus skip deletion parameter." "CRITICAL" tmp="$SKIP_DELETION"
exit 1 IFS=',' read -r -a SKIP_DELETION <<< "$tmp"
if [ $(arrayContains "${INITIATOR[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ] && [ $(arrayContains "${TARGET[$__type]}" "${SKIP_DELETION[@]}") -eq 0 ]; then
Logger "Bogus skip deletion parameter [$SKIP_DELETION]." "CRITICAL"
exit 1
fi
fi fi
} }
@ -1808,26 +1841,40 @@ function CreateStateDirs {
function _CheckLocksLocal { function _CheckLocksLocal {
local lockfile="${1}" local lockfile="${1}"
local replicaType="${2}"
local lockfile_content
local lock_pid
local lock_instance_id
if [ -f "$lockfile" ]; then local lockfileContent
lockfile_content=$(cat $lockfile) local lockPid
Logger "Master lock pid present: $lockfile_content" "DEBUG" local lockInstanceID
lock_pid=${lockfile_content%@*}
lock_instance_id=${lockfile_content#*@} if [ -s "$lockfile" ]; then
kill -9 $lock_pid > /dev/null 2>&1 lockfileContent=$(cat $lockfile)
Logger "Master lock pid present: $lockfileContent" "DEBUG"
lockPid=${lockfileContent%@*}
if [ $(IsInteger $lockPid) -ne 1 ]; then
Logger "Invalid pid [$lockPid] in local replica." "CRITICAL"
exit 1
fi
lockInstanceID=${lockfileContent#*@}
if [ "$lockInstanceID" == "" ]; then
Logger "Invalid instance id [$lockInstanceID] in local replica." "CRITICAL"
exit 1
Logger "Local $replicaType lock is: [$lockPid@$lockInstanceID]." "DEBUG"
fi
kill -0 $lockPid > /dev/null 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "There is a dead osync lock in [$lockfile]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a local dead osync lock [$lockPid@$lockInstanceID] that is no longer running. Resuming." "NOTICE"
#rm "$lockfile" if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
#if [ $? != 0 ]; then # REPLICA_OVERWRITE_LOCK disables noclobber option in WriteLock functions
# Logger "Cannot remove lock in [$lockfile]." "CRITICAL" INITIATOR_OVERWRITE_LOCK=true
# exit 1 elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
#fi TARGET_OVERWRITE_LOCK=true
fi
else else
Logger "There is already a local instance of osync running [$lock_pid] for this replica. Cannot start." "CRITICAL" Logger "There is already a local instance [$lockPid@$lockInstanceID] of osync running for this replica. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -1837,9 +1884,9 @@ function _CheckLocksRemote {
local lockfile="${1}" local lockfile="${1}"
local cmd local cmd
local lock_pid local lockPid
local lock_instance_id local lockInstanceID
local lockfile_content local lockfileContent
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
@ -1848,34 +1895,42 @@ function _CheckLocksRemote {
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
if [ $? != 0 ]; then if [ $? != 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then Logger "Cannot check remote replica lock." "CRITICAL"
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID) exit 1
else
Logger "Cannot get remote lockfile." "CRITICAL"
exit 1
fi
fi fi
lock_pid=${lockfile_content%@*} if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
lock_instance_id=${lockfile_content#*@} lockfileContent="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)"
if [ "$lock_pid" != "" ] && [ "$lock_instance_id" != "" ]; then lockPid=${lockfileContent%@*}
Logger "Remote lock is: $lock_pid@$lock_instance_id" "DEBUG" if [ $(IsInteger $lockPid) -ne 1 ]; then
Logger "Invalid pid [$lockPid] in remote replica lock." "CRITICAL"
exit 1
fi
lockInstanceID=${lockfileContent#*@}
if [ "$lockInstanceID" == "" ]; then
Logger "Invalid instance id [$lockInstanceID] in remote replica." "CRITICAL"
exit 1
fi
kill -0 $lock_pid > /dev/null 2>&1 Logger "Remote lock is: [$lockPid@$lockInstanceID]" "DEBUG"
kill -0 $lockPid > /dev/null 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
if [ "$lock_instance_id" == "$INSTANCE_ID" ]; then if [ "$lockInstanceID" == "$INSTANCE_ID" ]; then
Logger "There is a dead osync lock on target replica that corresponds to this initiator sync id [$lock_instance_id]. Instance [$lock_pid] no longer running. Resuming." "NOTICE" Logger "There is a remote dead osync lock [$lockPid@lockInstanceID] on target replica that corresponds to this initiator INSTANCE_ID. Pid [$lockPid] no longer running. Resuming." "NOTICE"
TARGET_OVERWRITE_LOCK=true
else else
if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]; then if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ]; then
Logger "WARNING: There is a dead osync lock on target replica that does not correspond to this initiator sync-id [$lock_instance_id]. Forcing resume." "WARN" Logger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Forcing resume." "WARN"
TARGET_OVERWRITE_LOCK=true
else else
Logger "There is a dead osync lock on target replica that does not correspond to this initiator sync-id [$lock_instance_id]. Will not resume." "CRITICAL" Logger "There is a remote (maybe dead) osync lock [$lockPid@$lockInstanceID] on target replica that does not correspond to this initiator INSTANCE_ID. Will not resume." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
else else
Logger "There is already a local instance of osync that locks target replica [$lock_pid@$lock_instance_id]. Cannot start." "CRITICAL" Logger "There is already a local instance of osync that locks target replica [$lockPid@$lockInstanceID]. Cannot start." "CRITICAL"
exit 1 exit 1
fi fi
fi fi
@ -1896,10 +1951,10 @@ function CheckLocks {
exit 1 exit 1
fi fi
else else
_CheckLocksLocal "${INITIATOR[$__lockFile]}" & _CheckLocksLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" &
pids="$!" pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_CheckLocksLocal "${TARGET[$__lockFile]}" & _CheckLocksLocal "${TARGET[$__lockFile]}" "${INITIATOR[$__type]}" &
pids="$pids;$!" pids="$pids;$!"
else else
_CheckLocksRemote "${TARGET[$__lockFile]}" & _CheckLocksRemote "${TARGET[$__lockFile]}" &
@ -1917,9 +1972,13 @@ function CheckLocks {
function _WriteLockFilesLocal { function _WriteLockFilesLocal {
local lockfile="${1}" local lockfile="${1}"
local replicaType="${2}" local replicaType="${2}"
local overwrite="${3:-false}"
( (
set -o noclobber if [ $overwrite == true ]; then
set -o noclobber
fi
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID" $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-$replicaType.$SCRIPT_PID"
) )
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -1934,18 +1993,22 @@ function _WriteLockFilesLocal {
function _WriteLockFilesRemote { function _WriteLockFilesRemote {
local lockfile="${1}" local lockfile="${1}"
local replicaType="${2}" local replicaType="${2}"
local overwrite="${3-false}"
local cmd local cmd
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
cmd=$SSH_CMD' "( set -o noclobber; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID' cmd=$SSH_CMD' "( if [ $overwrite == true ]; then set -o noclobber; fi; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID'
#WIP
#cmd=$SSH_CMD' "( set -o noclobber; echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\")" > /dev/null 2> $RUN_DIR/$PROGRAM._WriteLockFilesRemote.$replicaType.$SCRIPT_PID'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL" Logger "Could not create lock file on remote $replicaType in [$lockfile]." "CRITICAL"
Loggxer "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID)" "NOTICE"
return 1 return 1
else else
Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG" Logger "Locked remote $replicaType replica in [$lockfile]." "DEBUG"
@ -1959,14 +2022,14 @@ function WriteLockFiles {
local pidArray local pidArray
local pid local pid
_WriteLockFilesLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}"& _WriteLockFilesLocal "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" $INITIATOR_LOCK_OVERWRITE &
initiatorPid="$!" initiatorPid="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_WriteLockFilesLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & _WriteLockFilesLocal "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $TARGET_LOCK_OVERWRITE &
targetPid="$!" targetPid="$!"
else else
_WriteLockFilesRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" & _WriteLockFilesRemote "${TARGET[$__lockFile]}" "${TARGET[$__type]}" $TARGET_LOCK_OVERWRITE &
targetPid="$!" targetPid="$!"
fi fi
@ -2061,10 +2124,10 @@ function treeList {
local replicaType="${2}" # replica type: initiator, target local replicaType="${2}" # replica type: initiator, target
local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType)
local escapedReplicaPath local escapedReplicaPath
local rsyncCmd local rsyncCmd
escapedReplicaPath=$(EscapeSpaces "$replicaPath") escapedReplicaPath=$(EscapeSpaces "$replicaPath")
Logger "Creating $replicaType replica file list [$replicaPath]." "NOTICE" Logger "Creating $replicaType replica file list [$replicaPath]." "NOTICE"
@ -2102,6 +2165,17 @@ function deleteList {
local cmd local cmd
local failedDeletionListFromReplica
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
failedDeletionListFromReplica="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
failedDeletionListFromReplica="${INITIATOR[$__type]}"
else
Logger "Bogus replicaType in [${FUNCNAME[0]}]." "CRITICAL"
exit 1
fi
Logger "Creating $replicaType replica deleted file list." "NOTICE" Logger "Creating $replicaType replica deleted file list." "NOTICE"
if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeAfterFileNoSuffix]}" ]; then if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeAfterFileNoSuffix]}" ]; then
## Same functionnality, comm is much faster than grep but is not available on every platform ## Same functionnality, comm is much faster than grep but is not available on every platform
@ -2117,10 +2191,10 @@ function deleteList {
retval=$? retval=$?
# Add delete failed file list to current delete list and then empty it # Add delete failed file list to current delete list and then empty it
if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" ]; then if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$failedDeletionListFromReplica${INITIATOR[$__failedDeletedListFile]}" ]; then
cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$failedDeletionListFromReplica${INITIATOR[$__failedDeletedListFile]}" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
if [ $? == 0 ]; then if [ $? == 0 ]; then
rm -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}" rm -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$failedDeletionListFromReplica${INITIATOR[$__failedDeletedListFile]}"
else else
Logger "Cannot add failed deleted list to current deleted list for replica [$replicaType]." "ERROR" Logger "Cannot add failed deleted list to current deleted list for replica [$replicaType]." "ERROR"
fi fi
@ -2476,7 +2550,7 @@ function _deleteRemote {
exit 1 exit 1
fi fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETE_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _LOGGER_VERBOSE=$_LOGGER_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")" REPLICA_DIR="$(EscapeSpaces "$replicaDir")" SOFT_DELETE=$SOFT_DELETE DELETION_DIR="$(EscapeSpaces "$deletionDir")" FAILED_DELETE_LIST="$failedDeleteList" SUCCESS_DELETE_LIST="$successDeleteList" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1
## The following lines are executed remotely ## The following lines are executed remotely
function _logger { function _logger {
@ -2503,7 +2577,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
_logger "$prefix$value" _logger "$prefix$value"
return return
elif [ "$level" == "VERBOSE" ]; then elif [ "$level" == "VERBOSE" ]; then
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
_logger "$prefix$value" _logger "$prefix$value"
fi fi
return return
@ -2547,12 +2621,12 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
# In order to keep full path on soft deletion, create parent directories before move # In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")" parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then if [ "$parentdir" != "." ]; then
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE"
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE"
else else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
fi fi
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR" Logger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR"
@ -2566,10 +2640,9 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
else else
if [ $_DRYRUN == false ]; then if [ $_DRYRUN == false ]; then
if [ -e "$REPLICA_DIR$files" ]; then if [ -e "$REPLICA_DIR$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
$result=$?
Logger "Deleting [$REPLICA_DIR$files]." "VERBOSE" Logger "Deleting [$REPLICA_DIR$files]." "VERBOSE"
if [ $result != 0 ]; then $COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then
Logger "Cannot delete [$REPLICA_DIR$files]." "ERROR" Logger "Cannot delete [$REPLICA_DIR$files]." "ERROR"
echo "$files" >> "$FAILED_DELETE_LIST" echo "$files" >> "$FAILED_DELETE_LIST"
else else
@ -2816,8 +2889,7 @@ function Sync {
## Step 2 ## Step 2
if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
#TODO(high) only -X and -A (xattr & acl) should trigger this if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
if [ "$RSYNC_ATTR_ARGS" != "" ]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -2990,7 +3062,7 @@ function _SoftDeleteLocal {
Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE"
fi fi
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
$COMMAND_SUDO $FIND_CMD "$replicaDeletionPath/" -type f -ctime +$changeTime -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath/" -type f -ctime +$changeTime -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE"
@ -3040,7 +3112,7 @@ function _SoftDeleteRemote {
Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE"
fi fi
if [ $_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replicaDeletionPath'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type f -ctime +'$changeTime' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type d -empty -ctime '$changeTime' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replicaDeletionPath'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type f -ctime +'$changeTime' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replicaDeletionPath'/\" -type d -empty -ctime '$changeTime' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
@ -3287,7 +3359,7 @@ function Init {
## Set sync only function arguments for rsync ## Set sync only function arguments for rsync
SYNC_OPTS="-u" SYNC_OPTS="-u"
if [ $_VERBOSE == true ] || [ $_SUMMARY == true ]; then if [ $_LOGGER_VERBOSE == true ] || [ $_SUMMARY == true ]; then
SYNC_OPTS=$SYNC_OPTS" -i" SYNC_OPTS=$SYNC_OPTS" -i"
fi fi
@ -3300,8 +3372,8 @@ function Init {
## Conflict options ## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__backupDir]}\"" INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[$__backupDir]}\""
TARGET_BACKUP="--backup --backup-dir=\"${TARGET[$__replicaDir]}${TARGET[$__backupDir]}\"" TARGET_BACKUP="--backup --backup-dir=\"${TARGET[$__backupDir]}\""
if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then
INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
@ -3449,11 +3521,11 @@ for i in "$@"; do
opts=$opts" --dry" opts=$opts" --dry"
;; ;;
--silent) --silent)
_SILENT=true _LOGGER_SILENT=true
opts=$opts" --silent" opts=$opts" --silent"
;; ;;
--verbose) --verbose)
_VERBOSE=true _LOGGER_VERBOSE=true
opts=$opts" --verbose" opts=$opts" --verbose"
;; ;;
--stats) --stats)
@ -3501,20 +3573,17 @@ for i in "$@"; do
;; ;;
--skip-deletion=*) --skip-deletion=*)
opts=$opts" --skip-deletion=\"${i##*=}\"" opts=$opts" --skip-deletion=\"${i##*=}\""
IFS=',' read -r -a SKIP_DELETION <<< ${i##*=} SKIP_DELETION=${##*=}
;; ;;
--on-changes) --on-changes)
sync_on_changes=true sync_on_changes=true
_NOLOCKS=true _NOLOCKS=true
_LOGGER_PREFIX="date" _LOGGER_PREFIX="date"
_LOGGER_STDERR=true
;; ;;
--no-locks) --no-locks)
_NOLOCKS=true _NOLOCKS=true
;; ;;
--errors-only) --errors-only)
#TODO: let err_only only output to stderr
#_LOGGER_STDERR=true
_LOGGER_ERR_ONLY=true _LOGGER_ERR_ONLY=true
;; ;;
--summary) --summary)
@ -3569,6 +3638,18 @@ opts="${opts# *}"
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
if [ "$PRESERVE_ACL" == "" ]; then
PRESERVE_ACL="yes"
fi
if [ "$PRESERVE_XATTR" == "" ]; then
PRESERVE_XATTR="yes"
fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi
MIN_WAIT=30 MIN_WAIT=30
else else
ConfigFile="${1}" ConfigFile="${1}"