WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2019-11-02 14:45:23 +01:00
parent 22538907b3
commit 4d4728b90c
4 changed files with 309 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
dev/debug_osync.sh
View File

@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2019 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.3.0-prerc1
PROGRAM_BUILD=2019070901
PROGRAM_BUILD=2019110201
IS_STABLE=false
CONFIG_FILE_REVISION_REQUIRED=1.3.0
@ -42,8 +42,8 @@ CONFIG_FILE_REVISION_REQUIRED=1.3.0
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
# CleanUp no #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.3.0-dev-postRC2
_OFUNCTIONS_BUILD=20190701901
_OFUNCTIONS_VERSION=2.3.0-RC3
_OFUNCTIONS_BUILD=2019090901
_OFUNCTIONS_BOOTSTRAP=true
if ! type "$BASH" > /dev/null; then
@ -199,7 +199,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -284,8 +284,8 @@ function Logger {
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
@ -437,6 +437,7 @@ function CleanUp {
fi
}
function GenericTrapQuit {
local exitcode=0
@ -455,7 +456,6 @@ function GenericTrapQuit {
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
@ -556,7 +556,7 @@ function SendEmail {
fi
done
else
Logger "No valid email adresses given." "WARN"
Logger "No valid email addresses given." "WARN"
return 1
fi
@ -898,7 +898,7 @@ function ExecTasks {
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing commands indexed by pids
local commandsArrayOutput=() # Array contining command results indexed by pids
local commandsArrayOutput=() # Array containing command results indexed by pids
local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0
@ -930,6 +930,7 @@ function ExecTasks {
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3)
local randomOutputName # Random filename for command outputs
local currentRunningPids # String of pids running, used for debugging purposes only
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
@ -1043,6 +1044,11 @@ function ExecTasks {
function _ExecTasksPidsCheck {
newPidsArray=()
if [ "$currentRunningPids" != "$(joinString " " ${pidsArray[@]})" ]; then
Logger "ExecTask running for pids [$(joinString " " ${pidsArray[@]})]." "DEBUG"
currentRunningPids="$(joinString " " ${pidsArray[@]})"
fi
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
@ -1100,7 +1106,7 @@ function ExecTasks {
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
if [ -f "${commandsArrayOutput[$pid]}" ]; then
Logger "Command output was [$(cat "${commandsArrayOutput[$pid]}")\n]." "ERROR"
Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "ERROR"
fi
fi
errorcount=$((errorcount+1))
@ -1498,10 +1504,10 @@ function GetLocalOS {
# Get linux versions
if [ -f "/etc/os-release" ]; then
localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true)
localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true)
localOsName="$(GetConfFileValue "/etc/os-release" "NAME" true)"
localOsVer="$(GetConfFileValue "/etc/os-release" "VERSION" true)"
elif [ "$LOCAL_OS" == "BusyBox" ]; then
localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ')
localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')"
localOsName="BusyBox"
fi
@ -1605,7 +1611,7 @@ function GetRemoteOS {
local remoteOsVar
$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN="(o_0)"
function GetOs {
local localOsVar
@ -1640,7 +1646,7 @@ function GetOs {
localOsVer=$(grep "^VERSION=" "$osInfo")
localOsVer="${localOsVer##*=}"
elif [ "$localOsVar" == "BusyBox" ]; then
localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '`
localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ')
localOsName="BusyBox"
fi
@ -1677,14 +1683,14 @@ ENDSSH
if [ $? -ne 0 ]; then
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT] as [$REMOTE_USER]." "CRITICAL"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
Logger "$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
fi
exit 1
fi
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")
remoteOsVar="$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")"
case $remoteOsVar in
*"Android"*)
REMOTE_OS="Android"
@ -1743,7 +1749,6 @@ function RunLocalCommand {
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1752,7 +1757,7 @@ function RunLocalCommand {
fi
if [ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]; then
Logger "Command output:\n$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
fi
if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then
@ -1785,7 +1790,6 @@ function RunRemoteCommand {
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1795,7 +1799,7 @@ function RunRemoteCommand {
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ] && ([ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ])
then
Logger "Command output:\n$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
fi
if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then
@ -1820,7 +1824,6 @@ function RunBeforeHook {
fi
if [ "$pids" != "" ]; then
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1840,7 +1843,6 @@ function RunAfterHook {
fi
if [ "$pids" != "" ]; then
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1854,7 +1856,6 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != false ] && [ "$REMOTE_OPERATION" != false ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1879,7 +1880,6 @@ function CheckConnectivity3rdPartyHosts {
do
eval "$PING_CMD $i > /dev/null 2>&1" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -2003,6 +2003,11 @@ function PreInit {
SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\""
fi
## Optional SSH arguments
if [ "$SSH_OPTIONAL_ARGS" != "" ]; then
SSH_OPTS="$SSH_OPTS $SSH_OPTIONAL_ARGS"
fi
## Support for older config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]; then
RSYNC_EXECUTABLE=rsync
@ -2216,7 +2221,7 @@ function InitRemoteOSDependingSettings {
fi
if [ "$RSYNC_COMPRESS" == true ]; then
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx"
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip"
else
Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE"
fi
@ -2619,7 +2624,7 @@ function _CheckReplicasLocal {
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
return 1
else
Logger "Created local replica path [$replicaPath]." "NOTICE"
@ -2669,7 +2674,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -2804,7 +2809,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -2875,6 +2880,7 @@ function CleanUp {
fi
}
function _CheckReplicasRemoteSub {
if [ ! -d "$replicaPath" ]; then
if [ "$CREATE_DIRS" == true ]; then
@ -2926,7 +2932,7 @@ ENDSSH
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
(
_LOGGER_PREFIX=""
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
)
fi
if [ $retval -ne 0 ]; then
@ -2986,14 +2992,14 @@ function _HandleLocksLocal {
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
return 1
fi
fi
# Skip the whole part if overwrite true
if [ -s "$lockfile" ] && [ $overwrite != true ]; then
lockfileContent=$(cat $lockfile)
lockfileContent="$(head -c16384 $lockfile)"
Logger "Master lock pid present: $lockfileContent" "DEBUG"
lockPid="${lockfileContent%@*}"
if [ $(IsInteger $lockPid) -ne 1 ]; then
@ -3060,7 +3066,7 @@ env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_P
env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \
env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -3183,7 +3189,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -3254,6 +3260,7 @@ function CleanUp {
fi
}
function _HandleLocksRemoteSub {
local writeLocks=false
@ -3268,7 +3275,7 @@ function _HandleLocksRemoteSub {
# Skip the whole part if overwrite true
if [ -s "$lockfile" ] && [ $overwrite != true ]; then
lockfileContent=$(cat "$lockfile")
lockfileContent="$(head -c16384 "$lockfile")"
RemoteLogger "Master lock pid present: $lockfileContent" "DEBUG"
lockPid="${lockfileContent%@*}"
if [ $(IsInteger $lockPid) -ne 1 ]; then
@ -3334,7 +3341,7 @@ ENDSSH
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
(
_LOGGER_PREFIX=""
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
)
fi
if [ $retval -ne 0 ]; then
@ -3435,7 +3442,7 @@ $SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" env lockfile="'$lockfile'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
if [ -f "$lockfile" ]; then
rm -f "$lockfile"
fi
@ -3443,7 +3450,7 @@ ENDSSH
retval=$?
if [ $retval -ne 0 ]; then
Logger "Could not unlock $replicaType remote replica." "ERROR" $retval
Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
else
Logger "Removed remote $replicaType replica lock." "DEBUG"
fi
@ -3531,7 +3538,14 @@ function treeList {
retval=$?
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
mv -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "$treeFilename"
# mv fails on MacOS when $RUN_DIR =/tmp because of some shady apple BS
# see https://apple.stackexchange.com/questions/275521/how-does-group-wheel-get-on-my-files and #175
#mv -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "$treeFilename"
rm -f "$treeFileName" && cp -p "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "$treeFilename" && rm -f "$treeFileName"
if [ $? -ne 0 ]; then
Logger "Cannot move treeList files \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\" => \"$treeFilename\"". "ERROR"
return $retval
fi
fi
## Retval 24 = some files vanished while creating list
@ -3539,12 +3553,12 @@ function treeList {
return $?
elif [ $retval -eq 23 ]; then
Logger "Some files could not be listed in $replicaType replica [$replicaPath]. Check for failing symlinks." "ERROR" $retval
Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
return 0
else
Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
return $retval
fi
}
@ -3607,7 +3621,11 @@ function deleteList {
# Make sure deletion list does not contain duplicates from faledDeleteListFile
uniq "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
mv "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
#mv "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
rm -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" && cp -p "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\" => \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}\"" "ERROR"
fi
fi
return $retval
@ -3633,7 +3651,7 @@ function _getFileCtimeMtimeLocal {
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.subshellError.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Getting file time attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Command output:\n$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP")" "WARN"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP")" "WARN"
fi
return 1
else
@ -3662,7 +3680,7 @@ function _getFileCtimeMtimeRemote {
Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
fi
return $retval
fi
@ -3673,7 +3691,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP"
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -3808,7 +3826,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -3879,6 +3897,7 @@ function CleanUp {
fi
}
function _getFileCtimeMtimeRemoteSub {
while IFS='' read -r file; do
@ -3909,7 +3928,7 @@ ENDSSH
if [ $retval -ne 0 ]; then
Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
fi
return $retval
else
@ -4065,12 +4084,12 @@ function syncAttrs {
Logger "Getting list of files that need updates failed [$retval]. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated rsync output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
return $retval
else
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated list:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
fi
( grep -Ev "^[^ ]*(c|s|t)[^ ]* " "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP"
retval=$?
@ -4152,12 +4171,12 @@ function syncAttrs {
Logger "Updating file attributes on $destReplica [$retval]. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated rsync output:\n$(head -c16384 $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
return 1
else
if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated list:\n$(head -c16384 $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
fi
Logger "Successfully updated file attributes on $destReplica replica." "NOTICE"
fi
@ -4230,12 +4249,12 @@ function syncUpdate {
Logger "Updating $destinationReplica replica failed. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated rsync output:\n$(head -c16384 $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
exit 1
else
if [ -f "$RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated list:\n$(head -c16384 $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
fi
Logger "Updating $destinationReplica replica succeded." "NOTICE"
return 0
@ -4291,10 +4310,21 @@ function _deleteLocal {
if [ "$parentdir" != "." ]; then
mkdir -p "$replicaDir$deletionDir/$parentdir"
Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir/$parentdir] on $replicaType." "VERBOSE"
mv -f "$replicaDir$files" "$replicaDir$deletionDir/$parentdir"
#mv -f "$replicaDir$files" "$replicaDir$deletionDir/$parentdir"
#rm -f "$replicaDir$deletionDir/$parentdir"
cp -p "$replicaDir$files" "$replicaDir$deletionDir/$parentdir" && rm -f "$replicaDir$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$replicaDir$files\" => \"$replicaDir$deletionDir/$parentdir\"" "ERROR"
fi
else
Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir] on $replicaType." "VERBOSE"
mv -f "$replicaDir$files" "$replicaDir$deletionDir"
#mv -f "$replicaDir$files" "$replicaDir$deletionDir"
#rm -f "$replicaDir$deletionDir"
cp -p "$replicaDir$files" "$replicaDir$deletionDir" && rm -f "$replicaDir$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$replicaDir$files\" => \"$replicaDir$deletionDir\"" "ERROR"
fi
fi
retval=$?
if [ $retval -ne 0 ]; then
@ -4372,7 +4402,7 @@ function _deleteRemote {
Logger "Cannot copy the deletion list to remote replica." "ERROR" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP)" "ERROR"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP)" "ERROR"
fi
exit 1
fi
@ -4385,7 +4415,7 @@ env _DRYRUN="'$_DRYRUN'" \
env FILE_LIST="'${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}'" env REPLICA_DIR="'$replicaDir'" env SOFT_DELETE="'$SOFT_DELETE'" \
env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" env REPLICA_TYPE="'$replicaType'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -4471,7 +4501,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -4542,6 +4572,7 @@ function CleanUp {
fi
}
function _deleteRemoteSub {
## Empty earlier failed delete list
> "$FAILED_DELETE_LIST"
@ -4579,10 +4610,21 @@ function _deleteRemoteSub {
if [ "$parentdir" != "." ]; then
RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir] on $REPLICA_TYPE." "VERBOSE"
mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir"
mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
#mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
#rm -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
cp -p "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" && rm -f "$REPLICA_DIR$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$REPLICA_DIR$files\" => \"$REPLICA_DIR$DELETION_DIR/$parentdir\"" "ERROR"
fi
else
RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR] on $REPLICA_TYPE." "VERBOSE"
mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
#mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
#rm -f "$REPLICA_DIR$DELETION_DIR"
cp -p "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
rm -f "$REPLICA_DIR$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$REPLICA_DIR$files\" => \"$REPLICA_DIR$DELETION_DIR\"" "ERROR"
fi
fi
retval=$?
if [ $retval -ne 0 ]; then
@ -4627,8 +4669,8 @@ ENDSSH
retval=$?
if [ -s "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" ] && ([ $retval -ne 0 ] || [ "$_LOGGER_VERBOSE" == true ]); then
(
_LOGGER_PREFIX="RR"
Logger "$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP)" "ERROR"
_LOGGER_PREFIX=""
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP)" "ERROR"
)
fi
@ -4640,7 +4682,7 @@ ENDSSH
Logger "Cannot copy back the failed deletion list to initiator replica." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Comand output: $(cat $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
return 1
fi
@ -4776,20 +4818,20 @@ function Sync {
if [ "$RESUME_SYNC" != false ]; then
if [ -f "${INITIATOR[$__resumeCount]}" ]; then
resumeCount=$(cat "${INITIATOR[$__resumeCount]}")
resumeCount="$(head -c16384 "${INITIATOR[$__resumeCount]}")"
else
resumeCount=0
fi
if [ $resumeCount -lt $RESUME_TRY ]; then
if [ -f "${INITIATOR[$__initiatorLastActionFile]}" ]; then
resumeInitiator=$(cat "${INITIATOR[$__initiatorLastActionFile]}")
resumeInitiator="$(head -c16384 "${INITIATOR[$__initiatorLastActionFile]}")"
else
resumeInitiator="${SYNC_ACTION[9]}"
fi
if [ -f "${INITIATOR[$__targetLastActionFile]}" ]; then
resumeTarget=$(cat "${INITIATOR[$__targetLastActionFile]}")
resumeTarget="$(head -c16384 "${INITIATOR[$__targetLastActionFile]}")"
else
resumeTarget="${SYNC_ACTION[9]}"
fi
@ -5282,9 +5324,9 @@ function _SoftDeleteLocal {
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Error while executing cleanup on $replicaType replica." "ERROR" $retval
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
else
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "File cleanup complete on $replicaType replica." "NOTICE"
fi
@ -5324,7 +5366,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -5459,7 +5501,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -5530,6 +5572,7 @@ function CleanUp {
fi
}
function _SoftDeleteRemoteSub {
if [ -d "$replicaDeletionPath" ]; then
$REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteList.$replicaType.$SCRIPT_PID.$TSTAMP" 2>> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP"
@ -5550,10 +5593,10 @@ function _SoftDeleteRemoteSub {
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
RemoteLogger "Error while executing cleanup on $replicaType replica." "ERROR" $retval
RemoteLogger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
RemoteLogger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
exit 1
else
RemoteLogger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
RemoteLogger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
RemoteLogger "File cleanup complete on $replicaType replica." "NOTICE"
exit 0
fi
@ -5575,13 +5618,13 @@ ENDSSH
Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" $retval
(
_LOGGER_PREFIX=""
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
)
else
Logger "Cleanup complete on $replicaType replica." "NOTICE"
(
_LOGGER_PREFIX=""
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
)
fi
}
@ -5640,7 +5683,7 @@ function _TriggerInitiatorRunLocal {
echo "$INSTANCE_ID#$(date '+%Y%m%dT%H%M%S.%N')" >> "$PUSH_FILE" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
if [ $? -ne 0 ]; then
Logger "Could not notify local initiator of file changes." "ERROR"
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
return 1
else
Logger "Initiator of instance [$INSTANCE_ID] should be notified of file changes now." "NOTICE"
@ -5660,7 +5703,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env INSTANCE_ID="'$INSTANCE_ID'" env PUSH_FILE="'$(EscapeSpaces "${INITIATOR[$__replicaDir]}${INITIATOR[$__updateTriggerFile]}")'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -5746,7 +5789,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -5817,6 +5860,7 @@ function CleanUp {
fi
}
if [ -d $(dirname "$PUSH_FILE") ]; then
#WIP no %N on BSD (also in local)
echo "$INSTANCE_ID#$(date '+%Y%m%dT%H%M%S.%N')" >> "$PUSH_FILE"
@ -5831,8 +5875,8 @@ ENDSSH
Logger "Could not notifiy remote initiator of file changes." "ERROR"
Logger "SSH_CMD [$SSH_CMD]" "DEBUG"
(
_LOGGER_PREFIX="RR"
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
_LOGGER_PREFIX=""
Logger "$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
)
return 1
else
@ -5955,7 +5999,7 @@ function LogConflicts {
(
_LOGGER_PREFIX=""
Logger "$(cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")" "ALWAYS"
Logger "Truncated output:\n$(head -c16384 "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")" "ALWAYS"
)
Logger "There are $conflicts conflictual files." "ALWAYS"
@ -5965,7 +6009,7 @@ function LogConflicts {
if [ "$ALERT_CONFLICTS" == true ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")"
body="Truncated list of conflictual files:"$'\n'"$(head -c16384 "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
@ -6594,6 +6638,10 @@ if [ $_QUICK_SYNC -eq 2 ]; then
MAX_EXEC_TIME_PER_CMD_AFTER=0
fi
if [ "$RSYNC_COMPRESS" == "" ]; then
RSYNC_COMPRESS=true
fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi

18
install.sh
View File

@ -17,8 +17,8 @@ INSTANCE_ID="installer-$SCRIPT_BUILD"
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
## Please adapt this to fit your distro needs
_OFUNCTIONS_VERSION=2.3.0-dev-postRC2
_OFUNCTIONS_BUILD=20190701901
_OFUNCTIONS_VERSION=2.3.0-RC3
_OFUNCTIONS_BUILD=2019090901
_OFUNCTIONS_BOOTSTRAP=true
if ! type "$BASH" > /dev/null; then
@ -170,7 +170,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -250,8 +250,8 @@ function Logger {
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
@ -396,6 +396,7 @@ function CleanUp {
fi
}
function GenericTrapQuit {
local exitcode=0
@ -414,7 +415,6 @@ function GenericTrapQuit {
}
# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@ -572,10 +572,10 @@ function GetLocalOS {
# Get linux versions
if [ -f "/etc/os-release" ]; then
localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true)
localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true)
localOsName="$(GetConfFileValue "/etc/os-release" "NAME" true)"
localOsVer="$(GetConfFileValue "/etc/os-release" "VERSION" true)"
elif [ "$LOCAL_OS" == "BusyBox" ]; then
localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ')
localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')"
localOsName="BusyBox"
fi

50
osync-batch.sh
View File

@ -1,9 +1,9 @@
#!/usr/bin/env bash
SUBPROGRAM=osync
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2018 by Orsiris de Jong"
AUTHOR="(L) 2013-2019 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2018100201
PROGRAM_BUILD=2019090901
## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits
@ -34,9 +34,6 @@ elif [ -w /var/tmp ]; then
else
RUN_DIR=.
fi
trap TrapQuit TERM EXIT HUP QUIT
# No need to edit under this line ##############################################################
#### RemoteLogger SUBSET ####
@ -82,7 +79,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -163,8 +160,8 @@ function Logger {
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
@ -205,6 +202,41 @@ function Logger {
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}
function CleanUp {
# Exit controlmaster before it's socket gets deleted
if [ "$SSH_CONTROLMASTER" == true ] && [ "$SSH_CMD" != "" ]; then
$SSH_CMD -O exit
fi
if [ "$_DEBUG" != true ]; then
# Removing optional remote $RUN_DIR that goes into local $RUN_DIR
if [ -d "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" ]; then
rm -rf "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP"
fi
# Removing all temporary run files
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
function TrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
function CheckEnvironment {
## osync / obackup executable full path can be set here if it cannot be found on the system
@ -294,6 +326,8 @@ function Usage {
exit 128
}
trap TrapQuit TERM EXIT HUP QUIT
opts=""
for i in "$@"
do

210
osync.sh
View File

@ -7,14 +7,14 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2019 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.3.0-prerc1
PROGRAM_BUILD=2019070901
PROGRAM_BUILD=2019110201
IS_STABLE=false
CONFIG_FILE_REVISION_REQUIRED=1.3.0
_OFUNCTIONS_VERSION=2.3.0-dev-postRC2
_OFUNCTIONS_BUILD=20190701901
_OFUNCTIONS_VERSION=2.3.0-RC3
_OFUNCTIONS_BUILD=2019090901
_OFUNCTIONS_BOOTSTRAP=true
if ! type "$BASH" > /dev/null; then
@ -166,7 +166,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -246,8 +246,8 @@ function Logger {
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/env _REMOTE_TOKEN=__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
@ -392,6 +392,7 @@ function CleanUp {
fi
}
function GenericTrapQuit {
local exitcode=0
@ -410,7 +411,6 @@ function GenericTrapQuit {
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
@ -509,7 +509,7 @@ function SendEmail {
fi
done
else
Logger "No valid email adresses given." "WARN"
Logger "No valid email addresses given." "WARN"
return 1
fi
@ -832,7 +832,7 @@ function ExecTasks {
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid=() # Array containing commands indexed by pids
local commandsArrayOutput=() # Array contining command results indexed by pids
local commandsArrayOutput=() # Array containing command results indexed by pids
local postponedRetryCount=0 # Number of current postponed commands retries
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
local postponedCounter=0
@ -863,6 +863,7 @@ function ExecTasks {
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3)
local randomOutputName # Random filename for command outputs
local currentRunningPids # String of pids running, used for debugging purposes only
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
@ -975,6 +976,11 @@ function ExecTasks {
function _ExecTasksPidsCheck {
newPidsArray=()
if [ "$currentRunningPids" != "$(joinString " " ${pidsArray[@]})" ]; then
Logger "ExecTask running for pids [$(joinString " " ${pidsArray[@]})]." "DEBUG"
currentRunningPids="$(joinString " " ${pidsArray[@]})"
fi
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
@ -1032,7 +1038,7 @@ function ExecTasks {
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
if [ -f "${commandsArrayOutput[$pid]}" ]; then
Logger "Command output was [$(cat "${commandsArrayOutput[$pid]}")\n]." "ERROR"
Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "ERROR"
fi
fi
errorcount=$((errorcount+1))
@ -1422,10 +1428,10 @@ function GetLocalOS {
# Get linux versions
if [ -f "/etc/os-release" ]; then
localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true)
localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true)
localOsName="$(GetConfFileValue "/etc/os-release" "NAME" true)"
localOsVer="$(GetConfFileValue "/etc/os-release" "VERSION" true)"
elif [ "$LOCAL_OS" == "BusyBox" ]; then
localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ')
localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')"
localOsName="BusyBox"
fi
@ -1473,7 +1479,7 @@ function GetRemoteOS {
local remoteOsVar
$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN="(o_0)"
function GetOs {
local localOsVar
@ -1508,7 +1514,7 @@ function GetOs {
localOsVer=$(grep "^VERSION=" "$osInfo")
localOsVer="${localOsVer##*=}"
elif [ "$localOsVar" == "BusyBox" ]; then
localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '`
localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ')
localOsName="BusyBox"
fi
@ -1545,14 +1551,14 @@ ENDSSH
if [ $? -ne 0 ]; then
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT] as [$REMOTE_USER]." "CRITICAL"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
Logger "$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR"
fi
exit 1
fi
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")
remoteOsVar="$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")"
case $remoteOsVar in
*"Android"*)
REMOTE_OS="Android"
@ -1610,7 +1616,6 @@ function RunLocalCommand {
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1619,7 +1624,7 @@ function RunLocalCommand {
fi
if [ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]; then
Logger "Command output:\n$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
fi
if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then
@ -1651,7 +1656,6 @@ function RunRemoteCommand {
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1661,7 +1665,7 @@ function RunRemoteCommand {
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ] && ([ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ])
then
Logger "Command output:\n$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE"
fi
if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then
@ -1685,7 +1689,6 @@ function RunBeforeHook {
fi
if [ "$pids" != "" ]; then
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1704,7 +1707,6 @@ function RunAfterHook {
fi
if [ "$pids" != "" ]; then
ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1716,7 +1718,6 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != false ] && [ "$REMOTE_OPERATION" != false ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1738,7 +1739,6 @@ function CheckConnectivity3rdPartyHosts {
do
eval "$PING_CMD $i > /dev/null 2>&1" &
ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
#ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1857,6 +1857,11 @@ function PreInit {
SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\""
fi
## Optional SSH arguments
if [ "$SSH_OPTIONAL_ARGS" != "" ]; then
SSH_OPTS="$SSH_OPTS $SSH_OPTIONAL_ARGS"
fi
## Support for older config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]; then
RSYNC_EXECUTABLE=rsync
@ -2067,7 +2072,7 @@ function InitRemoteOSDependingSettings {
fi
if [ "$RSYNC_COMPRESS" == true ]; then
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx"
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip"
else
Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE"
fi
@ -2466,7 +2471,7 @@ function _CheckReplicasLocal {
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
return 1
else
Logger "Created local replica path [$replicaPath]." "NOTICE"
@ -2515,7 +2520,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -2646,7 +2651,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -2712,6 +2717,7 @@ function CleanUp {
fi
}
function _CheckReplicasRemoteSub {
if [ ! -d "$replicaPath" ]; then
if [ "$CREATE_DIRS" == true ]; then
@ -2763,7 +2769,7 @@ ENDSSH
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
(
_LOGGER_PREFIX=""
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
)
fi
if [ $retval -ne 0 ]; then
@ -2821,14 +2827,14 @@ function _HandleLocksLocal {
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
return 1
fi
fi
# Skip the whole part if overwrite true
if [ -s "$lockfile" ] && [ $overwrite != true ]; then
lockfileContent=$(cat $lockfile)
lockfileContent="$(head -c16384 $lockfile)"
Logger "Master lock pid present: $lockfileContent" "DEBUG"
lockPid="${lockfileContent%@*}"
if [ $(IsInteger $lockPid) -ne 1 ]; then
@ -2894,7 +2900,7 @@ env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_P
env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \
env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -3013,7 +3019,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -3079,6 +3085,7 @@ function CleanUp {
fi
}
function _HandleLocksRemoteSub {
local writeLocks=false
@ -3093,7 +3100,7 @@ function _HandleLocksRemoteSub {
# Skip the whole part if overwrite true
if [ -s "$lockfile" ] && [ $overwrite != true ]; then
lockfileContent=$(cat "$lockfile")
lockfileContent="$(head -c16384 "$lockfile")"
RemoteLogger "Master lock pid present: $lockfileContent" "DEBUG"
lockPid="${lockfileContent%@*}"
if [ $(IsInteger $lockPid) -ne 1 ]; then
@ -3159,7 +3166,7 @@ ENDSSH
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
(
_LOGGER_PREFIX=""
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "NOTICE"
)
fi
if [ $retval -ne 0 ]; then
@ -3257,7 +3264,7 @@ $SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" env lockfile="'$lockfile'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
if [ -f "$lockfile" ]; then
rm -f "$lockfile"
fi
@ -3265,7 +3272,7 @@ ENDSSH
retval=$?
if [ $retval -ne 0 ]; then
Logger "Could not unlock $replicaType remote replica." "ERROR" $retval
Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
else
Logger "Removed remote $replicaType replica lock." "DEBUG"
fi
@ -3351,7 +3358,14 @@ function treeList {
retval=$?
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
mv -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "$treeFilename"
# mv fails on MacOS when $RUN_DIR =/tmp because of some shady apple BS
# see https://apple.stackexchange.com/questions/275521/how-does-group-wheel-get-on-my-files and #175
#mv -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "$treeFilename"
rm -f "$treeFileName" && cp -p "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "$treeFilename" && rm -f "$treeFileName"
if [ $? -ne 0 ]; then
Logger "Cannot move treeList files \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\" => \"$treeFilename\"". "ERROR"
return $retval
fi
fi
## Retval 24 = some files vanished while creating list
@ -3359,12 +3373,12 @@ function treeList {
return $?
elif [ $retval -eq 23 ]; then
Logger "Some files could not be listed in $replicaType replica [$replicaPath]. Check for failing symlinks." "ERROR" $retval
Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
return 0
else
Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN"
return $retval
fi
}
@ -3426,7 +3440,11 @@ function deleteList {
# Make sure deletion list does not contain duplicates from faledDeleteListFile
uniq "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
mv "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
#mv "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
rm -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" && cp -p "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}" && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\" => \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}\"" "ERROR"
fi
fi
return $retval
@ -3451,7 +3469,7 @@ function _getFileCtimeMtimeLocal {
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.subshellError.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Getting file time attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Command output:\n$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP")" "WARN"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP")" "WARN"
fi
return 1
else
@ -3479,7 +3497,7 @@ function _getFileCtimeMtimeRemote {
Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
fi
return $retval
fi
@ -3490,7 +3508,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP"
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -3621,7 +3639,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -3687,6 +3705,7 @@ function CleanUp {
fi
}
function _getFileCtimeMtimeRemoteSub {
while IFS='' read -r file; do
@ -3717,7 +3736,7 @@ ENDSSH
if [ $retval -ne 0 ]; then
Logger "Getting file attributes failed [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
fi
return $retval
else
@ -3870,12 +3889,12 @@ function syncAttrs {
Logger "Getting list of files that need updates failed [$retval]. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated rsync output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
return $retval
else
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated list:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
fi
( grep -Ev "^[^ ]*(c|s|t)[^ ]* " "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP"
retval=$?
@ -3957,12 +3976,12 @@ function syncAttrs {
Logger "Updating file attributes on $destReplica [$retval]. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated rsync output:\n$(head -c16384 $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
return 1
else
if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated list:\n$(head -c16384 $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
fi
Logger "Successfully updated file attributes on $destReplica replica." "NOTICE"
fi
@ -4034,12 +4053,12 @@ function syncUpdate {
Logger "Updating $destinationReplica replica failed. Stopping execution." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated rsync output:\n$(head -c16384 $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
exit 1
else
if [ -f "$RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated list:\n$(head -c16384 $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
fi
Logger "Updating $destinationReplica replica succeded." "NOTICE"
return 0
@ -4094,10 +4113,21 @@ function _deleteLocal {
if [ "$parentdir" != "." ]; then
mkdir -p "$replicaDir$deletionDir/$parentdir"
Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir/$parentdir] on $replicaType." "VERBOSE"
mv -f "$replicaDir$files" "$replicaDir$deletionDir/$parentdir"
#mv -f "$replicaDir$files" "$replicaDir$deletionDir/$parentdir"
#rm -f "$replicaDir$deletionDir/$parentdir"
cp -p "$replicaDir$files" "$replicaDir$deletionDir/$parentdir" && rm -f "$replicaDir$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$replicaDir$files\" => \"$replicaDir$deletionDir/$parentdir\"" "ERROR"
fi
else
Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir] on $replicaType." "VERBOSE"
mv -f "$replicaDir$files" "$replicaDir$deletionDir"
#mv -f "$replicaDir$files" "$replicaDir$deletionDir"
#rm -f "$replicaDir$deletionDir"
cp -p "$replicaDir$files" "$replicaDir$deletionDir" && rm -f "$replicaDir$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$replicaDir$files\" => \"$replicaDir$deletionDir\"" "ERROR"
fi
fi
retval=$?
if [ $retval -ne 0 ]; then
@ -4174,7 +4204,7 @@ function _deleteRemote {
Logger "Cannot copy the deletion list to remote replica." "ERROR" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP" ]; then
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP)" "ERROR"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP)" "ERROR"
fi
exit 1
fi
@ -4187,7 +4217,7 @@ env _DRYRUN="'$_DRYRUN'" \
env FILE_LIST="'${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}'" env REPLICA_DIR="'$replicaDir'" env SOFT_DELETE="'$SOFT_DELETE'" \
env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" env REPLICA_TYPE="'$replicaType'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -4269,7 +4299,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -4335,6 +4365,7 @@ function CleanUp {
fi
}
function _deleteRemoteSub {
## Empty earlier failed delete list
> "$FAILED_DELETE_LIST"
@ -4372,10 +4403,21 @@ function _deleteRemoteSub {
if [ "$parentdir" != "." ]; then
RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir] on $REPLICA_TYPE." "VERBOSE"
mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir"
mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
#mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
#rm -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir"
cp -p "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" && rm -f "$REPLICA_DIR$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$REPLICA_DIR$files\" => \"$REPLICA_DIR$DELETION_DIR/$parentdir\"" "ERROR"
fi
else
RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR] on $REPLICA_TYPE." "VERBOSE"
mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
#mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
#rm -f "$REPLICA_DIR$DELETION_DIR"
cp -p "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR"
rm -f "$REPLICA_DIR$files"
if [ $? -ne 0 ]; then
Logger "Cannot move \"$REPLICA_DIR$files\" => \"$REPLICA_DIR$DELETION_DIR\"" "ERROR"
fi
fi
retval=$?
if [ $retval -ne 0 ]; then
@ -4420,8 +4462,8 @@ ENDSSH
retval=$?
if [ -s "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" ] && ([ $retval -ne 0 ] || [ "$_LOGGER_VERBOSE" == true ]); then
(
_LOGGER_PREFIX="RR"
Logger "$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP)" "ERROR"
_LOGGER_PREFIX=""
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP)" "ERROR"
)
fi
@ -4433,7 +4475,7 @@ ENDSSH
Logger "Cannot copy back the failed deletion list to initiator replica." "CRITICAL" $retval
_LOGGER_SILENT=true Logger "Command was [$rsyncCmd]." "WARN"
if [ -f "$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Comand output: $(cat $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP)" "NOTICE"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP)" "NOTICE"
fi
return 1
fi
@ -4566,20 +4608,20 @@ function Sync {
if [ "$RESUME_SYNC" != false ]; then
if [ -f "${INITIATOR[$__resumeCount]}" ]; then
resumeCount=$(cat "${INITIATOR[$__resumeCount]}")
resumeCount="$(head -c16384 "${INITIATOR[$__resumeCount]}")"
else
resumeCount=0
fi
if [ $resumeCount -lt $RESUME_TRY ]; then
if [ -f "${INITIATOR[$__initiatorLastActionFile]}" ]; then
resumeInitiator=$(cat "${INITIATOR[$__initiatorLastActionFile]}")
resumeInitiator="$(head -c16384 "${INITIATOR[$__initiatorLastActionFile]}")"
else
resumeInitiator="${SYNC_ACTION[9]}"
fi
if [ -f "${INITIATOR[$__targetLastActionFile]}" ]; then
resumeTarget=$(cat "${INITIATOR[$__targetLastActionFile]}")
resumeTarget="$(head -c16384 "${INITIATOR[$__targetLastActionFile]}")"
else
resumeTarget="${SYNC_ACTION[9]}"
fi
@ -5071,9 +5113,9 @@ function _SoftDeleteLocal {
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
Logger "Error while executing cleanup on $replicaType replica." "ERROR" $retval
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
else
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "File cleanup complete on $replicaType replica." "NOTICE"
fi
@ -5112,7 +5154,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -5243,7 +5285,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -5309,6 +5351,7 @@ function CleanUp {
fi
}
function _SoftDeleteRemoteSub {
if [ -d "$replicaDeletionPath" ]; then
$REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteList.$replicaType.$SCRIPT_PID.$TSTAMP" 2>> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP"
@ -5329,10 +5372,10 @@ function _SoftDeleteRemoteSub {
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then
RemoteLogger "Error while executing cleanup on $replicaType replica." "ERROR" $retval
RemoteLogger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
RemoteLogger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
exit 1
else
RemoteLogger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
RemoteLogger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.deleteErrors.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
RemoteLogger "File cleanup complete on $replicaType replica." "NOTICE"
exit 0
fi
@ -5354,13 +5397,13 @@ ENDSSH
Logger "Error while executing cleanup on remote $replicaType replica." "ERROR" $retval
(
_LOGGER_PREFIX=""
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN"
)
else
Logger "Cleanup complete on $replicaType replica." "NOTICE"
(
_LOGGER_PREFIX=""
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
Logger "Truncated output:\n$(head -c16384 $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE"
)
fi
}
@ -5417,7 +5460,7 @@ function _TriggerInitiatorRunLocal {
echo "$INSTANCE_ID#$(date '+%Y%m%dT%H%M%S.%N')" >> "$PUSH_FILE" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
if [ $? -ne 0 ]; then
Logger "Could not notify local initiator of file changes." "ERROR"
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
return 1
else
Logger "Initiator of instance [$INSTANCE_ID] should be notified of file changes now." "NOTICE"
@ -5436,7 +5479,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env INSTANCE_ID="'$INSTANCE_ID'" env PUSH_FILE="'$(EscapeSpaces "${INITIATOR[$__replicaDir]}${INITIATOR[$__updateTriggerFile]}")'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
_REMOTE_TOKEN=(o_0)
_REMOTE_TOKEN="(o_0)"
## Default directory where to store temporary run files
if [ -w /tmp ]; then
@ -5518,7 +5561,7 @@ function RemoteLogger {
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
prefix="RTIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
@ -5584,6 +5627,7 @@ function CleanUp {
fi
}
if [ -d $(dirname "$PUSH_FILE") ]; then
#WIP no %N on BSD (also in local)
echo "$INSTANCE_ID#$(date '+%Y%m%dT%H%M%S.%N')" >> "$PUSH_FILE"
@ -5598,8 +5642,8 @@ ENDSSH
Logger "Could not notifiy remote initiator of file changes." "ERROR"
Logger "SSH_CMD [$SSH_CMD]" "DEBUG"
(
_LOGGER_PREFIX="RR"
Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
_LOGGER_PREFIX=""
Logger "$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP")" "ERROR"
)
return 1
else
@ -5717,7 +5761,7 @@ function LogConflicts {
(
_LOGGER_PREFIX=""
Logger "$(cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")" "ALWAYS"
Logger "Truncated output:\n$(head -c16384 "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")" "ALWAYS"
)
Logger "There are $conflicts conflictual files." "ALWAYS"
@ -5727,7 +5771,7 @@ function LogConflicts {
if [ "$ALERT_CONFLICTS" == true ] && [ -s "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
subject="Conflictual files found in [$INSTANCE_ID]"
body="List of conflictual files:"$'\n'"$(cat "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")"
body="Truncated list of conflictual files:"$'\n'"$(head -c16384 "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}")"
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"
fi
@ -6352,6 +6396,10 @@ if [ $_QUICK_SYNC -eq 2 ]; then
MAX_EXEC_TIME_PER_CMD_AFTER=0
fi
if [ "$RSYNC_COMPRESS" == "" ]; then
RSYNC_COMPRESS=true
fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi