WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2016-10-19 12:30:20 +02:00
parent 6bae927a05
commit 8d98355867
2 changed files with 193 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
dev/debug_osync.sh
View File

@ -45,7 +45,7 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016091601 ## FUNC_BUILD=2016101901
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -148,7 +148,7 @@ function _Logger {
local evalue="${3}" # What to log to stderr local evalue="${3}" # What to log to stderr
echo -e "$lvalue" >> "$LOG_FILE" echo -e "$lvalue" >> "$LOG_FILE"
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" #WIP CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then
cat <<< "$evalue" 1>&2 cat <<< "$evalue" 1>&2
@ -581,6 +581,7 @@ function TrapError {
local job="$0" local job="$0"
local line="$1" local line="$1"
local code="${2:-1}" local code="${2:-1}"
if [ $_SILENT == false ]; then if [ $_SILENT == false ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi fi
@ -588,6 +589,7 @@ function TrapError {
function LoadConfigFile { function LoadConfigFile {
local configFile="${1}" local configFile="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -651,13 +653,13 @@ function joinString {
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local caller_name="${4}" # Who called this function local callerName="${4}" # Who called this function
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false
local keep_logging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging
Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
@ -693,8 +695,8 @@ function WaitForTaskCompletion {
exec_time=$SECONDS exec_time=$SECONDS
fi fi
if [ $keep_logging -ne 0 ]; then if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keep_logging)) -eq 0 ]; then if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time log_ttime=$exec_time
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
@ -702,15 +704,15 @@ function WaitForTaskCompletion {
fi fi
fi fi
if [ $exec_time -gt $soft_max_time ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert == true ] && [ $soft_max_time -ne 0 ]; then if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true soft_alert=true
SendAlert true SendAlert true
fi fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
KillChilds $pid true KillChilds $pid true
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -738,7 +740,7 @@ function WaitForTaskCompletion {
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
errorcount=$((errorcount+1)) errorcount=$((errorcount+1))
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG" Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then
WAIT_FOR_TASK_COMPLETION="$pid:$retval" WAIT_FOR_TASK_COMPLETION="$pid:$retval"
else else
@ -759,7 +761,7 @@ function WaitForTaskCompletion {
sleep $SLEEP_TIME sleep $SLEEP_TIME
done done
Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors # Return exit code if only one process was monitored, else return number of errors
if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then
@ -776,8 +778,13 @@ function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ? local readFromFile="${3:-false}" # Is commandsArg a file or a string ?
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local callerName="${6}" # Who called this function
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local commandCount local commandCount
local command local command
@ -816,7 +823,7 @@ function ParallelExec {
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
fi fi
Logger "Running command [$command]." "DEBUG" Logger "Running command [$command]." "DEBUG"
eval "$command" & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
pid=$! pid=$!
pidsArray+=($pid) pidsArray+=($pid)
commandsArrayPid[$pid]="$command" commandsArrayPid[$pid]="$command"
@ -876,6 +883,7 @@ function SedStripQuotes {
# Usage: var=$(StripSingleQuotes "$var") # Usage: var=$(StripSingleQuotes "$var")
function StripSingleQuotes { function StripSingleQuotes {
local string="${1}" local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string" echo "$string"
@ -884,6 +892,7 @@ function StripSingleQuotes {
# Usage: var=$(StripDoubleQuotes "$var") # Usage: var=$(StripDoubleQuotes "$var")
function StripDoubleQuotes { function StripDoubleQuotes {
local string="${1}" local string="${1}"
string="${string/#\"/}" string="${string/#\"/}"
string="${string/%\"/}" string="${string/%\"/}"
echo "$string" echo "$string"
@ -891,12 +900,14 @@ function StripDoubleQuotes {
function StripQuotes { function StripQuotes {
local string="${1}" local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
} }
# Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")" # Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")"
function EscapeSpaces { function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped local string="${1}" # String on which spaces will be escaped
echo "${string// /\\ }" echo "${string// /\\ }"
} }
@ -904,6 +915,7 @@ function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$" local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then if [[ $value =~ $re ]]; then
echo 1 echo 1
else else
@ -950,25 +962,25 @@ function urlEncode {
} }
function urlDecode { function urlDecode {
local url_encoded="${1//+/ }" local urlEncoded="${1//+/ }"
printf '%b' "${url_encoded//%/\\x}" printf '%b' "${urlEncoded//%/\\x}"
} }
function GetLocalOS { function GetLocalOS {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local local_os_var= local localOsVar
local_os_var="$(uname -spio 2>&1)" localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
local_os_var="$(uname -v 2>&1)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
local_os_var="$(uname)" localOsVar="$(uname)"
fi fi
fi fi
case $local_os_var in case $localOsVar in
*"Linux"*) *"Linux"*)
LOCAL_OS="Linux" LOCAL_OS="Linux"
;; ;;
@ -983,14 +995,14 @@ function GetLocalOS {
;; ;;
*) *)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option
Logger "Running on unknown local OS [$local_os_var]." "WARN" Logger "Running on unknown local OS [$localOsVar]." "WARN"
return return
fi fi
Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR"
exit 1 exit 1
;; ;;
esac esac
Logger "Local OS: [$local_os_var]." "DEBUG" Logger "Local OS: [$localOsVar]." "DEBUG"
} }
#### MINIMAL-FUNCTION-SET END #### #### MINIMAL-FUNCTION-SET END ####
@ -998,8 +1010,8 @@ function GetLocalOS {
function GetRemoteOS { function GetRemoteOS {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd= local cmd
local remote_os_var= local remoteOsVar
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
@ -1028,9 +1040,9 @@ function GetRemoteOS {
fi fi
fi fi
remote_os_var=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID")
case $remote_os_var in case $remoteOsVar in
*"Linux"*) *"Linux"*)
REMOTE_OS="Linux" REMOTE_OS="Linux"
;; ;;
@ -1049,21 +1061,21 @@ function GetRemoteOS {
;; ;;
*) *)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option
Logger "Running on unknown remote OS [$remote_os_var]." "WARN" Logger "Running on unknown remote OS [$remoteOsVar]." "WARN"
return return
fi fi
Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL"
Logger "Remote OS said:\n$remote_os_var" "CRITICAL" Logger "Remote OS said:\n$remoteOsVar" "CRITICAL"
exit 1 exit 1
esac esac
Logger "Remote OS: [$remote_os_var]." "DEBUG" Logger "Remote OS: [$remoteOsVar]." "DEBUG"
fi fi
} }
function RunLocalCommand { function RunLocalCommand {
local command="${1}" # Command to run local command="${1}" # Command to run
local hard_max_time="${2}" # Max time to wait for command to compleet local hardMaxTime="${2}" # Max time to wait for command to compleet
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ $_DRYRUN == true ]; then if [ $_DRYRUN == true ]; then
@ -1073,7 +1085,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1094,7 +1106,7 @@ function RunLocalCommand {
## Runs remote command $1 and waits for completition in $2 seconds ## Runs remote command $1 and waits for completition in $2 seconds
function RunRemoteCommand { function RunRemoteCommand {
local command="${1}" # Command to run local command="${1}" # Command to run
local hard_max_time="${2}" # Max time to wait for command to compleet local hardMaxTime="${2}" # Max time to wait for command to compleet
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
@ -1108,7 +1120,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1130,7 +1142,7 @@ function RunRemoteCommand {
function RunBeforeHook { function RunBeforeHook {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pids= local pids
if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then
RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
@ -1187,13 +1199,13 @@ function CheckConnectivityRemoteHost {
function CheckConnectivity3rdPartyHosts { function CheckConnectivity3rdPartyHosts {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local remote_3rd_party_success local remote3rdPartySuccess
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
remote_3rd_party_success=false remote3rdPartySuccess=false
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
@ -1202,11 +1214,11 @@ function CheckConnectivity3rdPartyHosts {
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
else else
remote_3rd_party_success=true remote3rdPartySuccess=true
fi fi
done done
if [ $remote_3rd_party_success == false ]; then if [ $remote3rdPartySuccess == false ]; then
Logger "No remote 3rd party host responded to ping. No internet ?" "WARN" Logger "No remote 3rd party host responded to ping. No internet ?" "WARN"
return 1 return 1
else else
@ -1269,7 +1281,7 @@ function __CheckArguments {
#__END_WITH_PARANOIA_DEBUG #__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd { function RsyncPatternsAdd {
local pattern_type="${1}" # exclude or include local patternType="${1}" # exclude or include
local pattern="${2}" local pattern="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -1281,7 +1293,7 @@ function RsyncPatternsAdd {
while [ -n "$rest" ] while [ -n "$rest" ]
do do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR # Take the string until first occurence until $PATH_SEPARATOR_CHAR
str=${rest%%;*} str=${rest%%;*} #TODO: replace ; with $PATH_SEPARATOR_CHAR
# Handle the last case # Handle the last case
if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then
rest= rest=
@ -1290,26 +1302,26 @@ function RsyncPatternsAdd {
rest=${rest#*$PATH_SEPARATOR_CHAR} rest=${rest#*$PATH_SEPARATOR_CHAR}
fi fi
if [ "$RSYNC_PATTERNS" == "" ]; then if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" RSYNC_PATTERNS="--"$patternType"=\"$str\""
else else
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\""
fi fi
done done
set +f set +f
} }
function RsyncPatternsFromAdd { function RsyncPatternsFromAdd {
local pattern_type="${1}" local patternType="${1}"
local pattern_from="${2}" local patternFrom="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## Check if the exclude list has a full path, and if not, add the config file path if there is one ## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then if [ "$(basename $patternFrom)" == "$patternFrom" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" patternFrom="$(dirname $CONFIG_FILE)/$patternFrom"
fi fi
if [ -e "$pattern_from" ]; then if [ -e "$patternFrom" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\""
fi fi
} }
@ -1473,9 +1485,20 @@ function PostInit {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
# Define remote commands # Define remote commands
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
else
SSH_PASSWORD=""
SSH_CMD=""
SCP_CMD=""
RSYNC_SSH_CMD=""
fi
} }
function InitLocalOSSettings { function InitLocalOSSettings {
@ -1621,6 +1644,11 @@ function CheckEnvironment {
Logger "ssh not present. Cannot start sync." "CRITICAL" Logger "ssh not present. Cannot start sync." "CRITICAL"
exit 1 exit 1
fi fi
if ! type sshpass > /dev/null 2>&1 ; then
Logger "sshpass not present. Cannot use password authentication." "CRITICAL"
exit 1
fi
fi fi
if ! type rsync > /dev/null 2>&1 ; then if ! type rsync > /dev/null 2>&1 ; then
@ -1666,8 +1694,8 @@ function CheckCurrentConfigAll {
fi fi
#TODO(low): Add runtime variable tests (RSYNC_ARGS etc) #TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then if [ "$REMOTE_OPERATION" == "yes" ] && ([ ! -f "$SSH_RSA_PRIVATE_KEY" ] && [ ! -f "$SSH_PASSWORD_FILE" ]); then
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL" Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL"
exit 1 exit 1
fi fi
} }
@ -3163,7 +3191,10 @@ function Init {
fi fi
if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa if [ ! -f "$SSH_PASSWORD_FILE" ]; then
# Assume that there might exist a standard rsa key
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
fi
fi fi
# remove everything before '@' # remove everything before '@'
@ -3194,7 +3225,7 @@ function Init {
INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR/#\~/$HOME}" INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR/#\~/$HOME}"
TARGET_SYNC_DIR="${TARGET_SYNC_DIR/#\~/$HOME}" TARGET_SYNC_DIR="${TARGET_SYNC_DIR/#\~/$HOME}"
SSH_RSA_PRIVATE_KEY="${SSH_RSA_PRIVATE_KEY/#\~/$HOME}" SSH_RSA_PRIVATE_KEY="${SSH_RSA_PRIVATE_KEY/#\~/$HOME}"
SSH_PASSWORD_FILE="${SSH_PASSWORD_FILE/#\~/$HOME}"
## Replica format ## Replica format
## Why the f*** does bash not have simple objects ? ## Why the f*** does bash not have simple objects ?
@ -3332,6 +3363,7 @@ function Usage {
echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)" echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)"
echo "--target=\"\" Local or remote target replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/target/replica (is mandatory)" echo "--target=\"\" Local or remote target replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/target/replica (is mandatory)"
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica" echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica"
echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used"
echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets" echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets"
echo "" echo ""
echo "Additionaly, you may set most osync options at runtime. eg:" echo "Additionaly, you may set most osync options at runtime. eg:"
@ -3461,6 +3493,10 @@ for i in "$@"; do
SSH_RSA_PRIVATE_KEY=${i##*=} SSH_RSA_PRIVATE_KEY=${i##*=}
opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\"" opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\""
;; ;;
--password-file=*)
SSH_PASSWORD_FILE=${i##*=}
opts=$opts" --password-file\"$SSH_PASSWORD_FILE\""
;;
--instance-id=*) --instance-id=*)
INSTANCE_ID=${i##*=} INSTANCE_ID=${i##*=}
opts=$opts" --instance-id=\"$INSTANCE_ID\"" opts=$opts" --instance-id=\"$INSTANCE_ID\""
@ -3546,9 +3582,9 @@ opts="${opts# *}"
GetLocalOS GetLocalOS
InitLocalOSSettings InitLocalOSSettings
CheckEnvironment
PreInit PreInit
Init Init
CheckEnvironment
PostInit PostInit
if [ $_QUICK_SYNC -lt 2 ]; then if [ $_QUICK_SYNC -lt 2 ]; then
CheckCurrentConfig CheckCurrentConfig

154
osync.sh
View File

@ -11,7 +11,7 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016091601 ## FUNC_BUILD=2016101901
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -109,7 +109,7 @@ function _Logger {
local evalue="${3}" # What to log to stderr local evalue="${3}" # What to log to stderr
echo -e "$lvalue" >> "$LOG_FILE" echo -e "$lvalue" >> "$LOG_FILE"
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" #WIP CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then if [ $_LOGGER_STDERR == true ] && [ "$evalue" != "" ]; then
cat <<< "$evalue" 1>&2 cat <<< "$evalue" 1>&2
@ -531,6 +531,7 @@ function TrapError {
local job="$0" local job="$0"
local line="$1" local line="$1"
local code="${2:-1}" local code="${2:-1}"
if [ $_SILENT == false ]; then if [ $_SILENT == false ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi fi
@ -540,6 +541,7 @@ function LoadConfigFile {
local configFile="${1}" local configFile="${1}"
if [ ! -f "$configFile" ]; then if [ ! -f "$configFile" ]; then
Logger "Cannot load configuration file [$configFile]. Cannot start." "CRITICAL" Logger "Cannot load configuration file [$configFile]. Cannot start." "CRITICAL"
exit 1 exit 1
@ -600,11 +602,11 @@ function joinString {
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local caller_name="${4}" # Who called this function local callerName="${4}" # Who called this function
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false
local keep_logging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
@ -639,8 +641,8 @@ function WaitForTaskCompletion {
exec_time=$SECONDS exec_time=$SECONDS
fi fi
if [ $keep_logging -ne 0 ]; then if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keep_logging)) -eq 0 ]; then if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time log_ttime=$exec_time
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
@ -648,15 +650,15 @@ function WaitForTaskCompletion {
fi fi
fi fi
if [ $exec_time -gt $soft_max_time ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert == true ] && [ $soft_max_time -ne 0 ]; then if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true soft_alert=true
SendAlert true SendAlert true
fi fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
KillChilds $pid true KillChilds $pid true
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -684,7 +686,7 @@ function WaitForTaskCompletion {
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
errorcount=$((errorcount+1)) errorcount=$((errorcount+1))
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG" Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then
WAIT_FOR_TASK_COMPLETION="$pid:$retval" WAIT_FOR_TASK_COMPLETION="$pid:$retval"
else else
@ -717,6 +719,11 @@ function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ? local readFromFile="${3:-false}" # Is commandsArg a file or a string ?
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local callerName="${6}" # Who called this function
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging
local commandCount local commandCount
@ -755,7 +762,7 @@ function ParallelExec {
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
fi fi
Logger "Running command [$command]." "DEBUG" Logger "Running command [$command]." "DEBUG"
eval "$command" & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
pid=$! pid=$!
pidsArray+=($pid) pidsArray+=($pid)
commandsArrayPid[$pid]="$command" commandsArrayPid[$pid]="$command"
@ -810,6 +817,7 @@ function SedStripQuotes {
# Usage: var=$(StripSingleQuotes "$var") # Usage: var=$(StripSingleQuotes "$var")
function StripSingleQuotes { function StripSingleQuotes {
local string="${1}" local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string" echo "$string"
@ -818,6 +826,7 @@ function StripSingleQuotes {
# Usage: var=$(StripDoubleQuotes "$var") # Usage: var=$(StripDoubleQuotes "$var")
function StripDoubleQuotes { function StripDoubleQuotes {
local string="${1}" local string="${1}"
string="${string/#\"/}" string="${string/#\"/}"
string="${string/%\"/}" string="${string/%\"/}"
echo "$string" echo "$string"
@ -825,12 +834,14 @@ function StripDoubleQuotes {
function StripQuotes { function StripQuotes {
local string="${1}" local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
} }
# Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")" # Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")"
function EscapeSpaces { function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped local string="${1}" # String on which spaces will be escaped
echo "${string// /\\ }" echo "${string// /\\ }"
} }
@ -838,6 +849,7 @@ function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$" local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then if [[ $value =~ $re ]]; then
echo 1 echo 1
else else
@ -884,24 +896,24 @@ function urlEncode {
} }
function urlDecode { function urlDecode {
local url_encoded="${1//+/ }" local urlEncoded="${1//+/ }"
printf '%b' "${url_encoded//%/\\x}" printf '%b' "${urlEncoded//%/\\x}"
} }
function GetLocalOS { function GetLocalOS {
local local_os_var= local localOsVar
local_os_var="$(uname -spio 2>&1)" localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
local_os_var="$(uname -v 2>&1)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
local_os_var="$(uname)" localOsVar="$(uname)"
fi fi
fi fi
case $local_os_var in case $localOsVar in
*"Linux"*) *"Linux"*)
LOCAL_OS="Linux" LOCAL_OS="Linux"
;; ;;
@ -916,22 +928,22 @@ function GetLocalOS {
;; ;;
*) *)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option
Logger "Running on unknown local OS [$local_os_var]." "WARN" Logger "Running on unknown local OS [$localOsVar]." "WARN"
return return
fi fi
Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR"
exit 1 exit 1
;; ;;
esac esac
Logger "Local OS: [$local_os_var]." "DEBUG" Logger "Local OS: [$localOsVar]." "DEBUG"
} }
#### MINIMAL-FUNCTION-SET END #### #### MINIMAL-FUNCTION-SET END ####
function GetRemoteOS { function GetRemoteOS {
local cmd= local cmd
local remote_os_var= local remoteOsVar
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
@ -960,9 +972,9 @@ function GetRemoteOS {
fi fi
fi fi
remote_os_var=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID")
case $remote_os_var in case $remoteOsVar in
*"Linux"*) *"Linux"*)
REMOTE_OS="Linux" REMOTE_OS="Linux"
;; ;;
@ -981,21 +993,21 @@ function GetRemoteOS {
;; ;;
*) *)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option
Logger "Running on unknown remote OS [$remote_os_var]." "WARN" Logger "Running on unknown remote OS [$remoteOsVar]." "WARN"
return return
fi fi
Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL"
Logger "Remote OS said:\n$remote_os_var" "CRITICAL" Logger "Remote OS said:\n$remoteOsVar" "CRITICAL"
exit 1 exit 1
esac esac
Logger "Remote OS: [$remote_os_var]." "DEBUG" Logger "Remote OS: [$remoteOsVar]." "DEBUG"
fi fi
} }
function RunLocalCommand { function RunLocalCommand {
local command="${1}" # Command to run local command="${1}" # Command to run
local hard_max_time="${2}" # Max time to wait for command to compleet local hardMaxTime="${2}" # Max time to wait for command to compleet
if [ $_DRYRUN == true ]; then if [ $_DRYRUN == true ]; then
Logger "Dryrun: Local command [$command] not run." "NOTICE" Logger "Dryrun: Local command [$command] not run." "NOTICE"
@ -1004,7 +1016,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1025,7 +1037,7 @@ function RunLocalCommand {
## Runs remote command $1 and waits for completition in $2 seconds ## Runs remote command $1 and waits for completition in $2 seconds
function RunRemoteCommand { function RunRemoteCommand {
local command="${1}" # Command to run local command="${1}" # Command to run
local hard_max_time="${2}" # Max time to wait for command to compleet local hardMaxTime="${2}" # Max time to wait for command to compleet
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
@ -1038,7 +1050,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hard_max_time ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1059,7 +1071,7 @@ function RunRemoteCommand {
function RunBeforeHook { function RunBeforeHook {
local pids= local pids
if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then
RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
@ -1113,13 +1125,13 @@ function CheckConnectivityRemoteHost {
function CheckConnectivity3rdPartyHosts { function CheckConnectivity3rdPartyHosts {
local remote_3rd_party_success local remote3rdPartySuccess
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
remote_3rd_party_success=false remote3rdPartySuccess=false
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
@ -1128,11 +1140,11 @@ function CheckConnectivity3rdPartyHosts {
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
else else
remote_3rd_party_success=true remote3rdPartySuccess=true
fi fi
done done
if [ $remote_3rd_party_success == false ]; then if [ $remote3rdPartySuccess == false ]; then
Logger "No remote 3rd party host responded to ping. No internet ?" "WARN" Logger "No remote 3rd party host responded to ping. No internet ?" "WARN"
return 1 return 1
else else
@ -1146,7 +1158,7 @@ function CheckConnectivity3rdPartyHosts {
#__END_WITH_PARANOIA_DEBUG #__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd { function RsyncPatternsAdd {
local pattern_type="${1}" # exclude or include local patternType="${1}" # exclude or include
local pattern="${2}" local pattern="${2}"
local rest local rest
@ -1157,7 +1169,7 @@ function RsyncPatternsAdd {
while [ -n "$rest" ] while [ -n "$rest" ]
do do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR # Take the string until first occurence until $PATH_SEPARATOR_CHAR
str=${rest%%;*} str=${rest%%;*} #TODO: replace ; with $PATH_SEPARATOR_CHAR
# Handle the last case # Handle the last case
if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then
rest= rest=
@ -1166,25 +1178,25 @@ function RsyncPatternsAdd {
rest=${rest#*$PATH_SEPARATOR_CHAR} rest=${rest#*$PATH_SEPARATOR_CHAR}
fi fi
if [ "$RSYNC_PATTERNS" == "" ]; then if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" RSYNC_PATTERNS="--"$patternType"=\"$str\""
else else
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\""
fi fi
done done
set +f set +f
} }
function RsyncPatternsFromAdd { function RsyncPatternsFromAdd {
local pattern_type="${1}" local patternType="${1}"
local pattern_from="${2}" local patternFrom="${2}"
## Check if the exclude list has a full path, and if not, add the config file path if there is one ## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then if [ "$(basename $patternFrom)" == "$patternFrom" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" patternFrom="$(dirname $CONFIG_FILE)/$patternFrom"
fi fi
if [ -e "$pattern_from" ]; then if [ -e "$patternFrom" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\""
fi fi
} }
@ -1345,9 +1357,20 @@ function PreInit {
function PostInit { function PostInit {
# Define remote commands # Define remote commands
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
else
SSH_PASSWORD=""
SSH_CMD=""
SCP_CMD=""
RSYNC_SSH_CMD=""
fi
} }
function InitLocalOSSettings { function InitLocalOSSettings {
@ -1490,6 +1513,11 @@ function CheckEnvironment {
Logger "ssh not present. Cannot start sync." "CRITICAL" Logger "ssh not present. Cannot start sync." "CRITICAL"
exit 1 exit 1
fi fi
if ! type sshpass > /dev/null 2>&1 ; then
Logger "sshpass not present. Cannot use password authentication." "CRITICAL"
exit 1
fi
fi fi
if ! type rsync > /dev/null 2>&1 ; then if ! type rsync > /dev/null 2>&1 ; then
@ -1533,8 +1561,8 @@ function CheckCurrentConfigAll {
fi fi
#TODO(low): Add runtime variable tests (RSYNC_ARGS etc) #TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then if [ "$REMOTE_OPERATION" == "yes" ] && ([ ! -f "$SSH_RSA_PRIVATE_KEY" ] && [ ! -f "$SSH_PASSWORD_FILE" ]); then
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL" Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL"
exit 1 exit 1
fi fi
} }
@ -2998,7 +3026,10 @@ function Init {
fi fi
if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa if [ ! -f "$SSH_PASSWORD_FILE" ]; then
# Assume that there might exist a standard rsa key
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
fi
fi fi
# remove everything before '@' # remove everything before '@'
@ -3029,7 +3060,7 @@ function Init {
INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR/#\~/$HOME}" INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR/#\~/$HOME}"
TARGET_SYNC_DIR="${TARGET_SYNC_DIR/#\~/$HOME}" TARGET_SYNC_DIR="${TARGET_SYNC_DIR/#\~/$HOME}"
SSH_RSA_PRIVATE_KEY="${SSH_RSA_PRIVATE_KEY/#\~/$HOME}" SSH_RSA_PRIVATE_KEY="${SSH_RSA_PRIVATE_KEY/#\~/$HOME}"
SSH_PASSWORD_FILE="${SSH_PASSWORD_FILE/#\~/$HOME}"
## Replica format ## Replica format
## Why the f*** does bash not have simple objects ? ## Why the f*** does bash not have simple objects ?
@ -3165,6 +3196,7 @@ function Usage {
echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)" echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)"
echo "--target=\"\" Local or remote target replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/target/replica (is mandatory)" echo "--target=\"\" Local or remote target replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/target/replica (is mandatory)"
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica" echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica"
echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used"
echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets" echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets"
echo "" echo ""
echo "Additionaly, you may set most osync options at runtime. eg:" echo "Additionaly, you may set most osync options at runtime. eg:"
@ -3293,6 +3325,10 @@ for i in "$@"; do
SSH_RSA_PRIVATE_KEY=${i##*=} SSH_RSA_PRIVATE_KEY=${i##*=}
opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\"" opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\""
;; ;;
--password-file=*)
SSH_PASSWORD_FILE=${i##*=}
opts=$opts" --password-file\"$SSH_PASSWORD_FILE\""
;;
--instance-id=*) --instance-id=*)
INSTANCE_ID=${i##*=} INSTANCE_ID=${i##*=}
opts=$opts" --instance-id=\"$INSTANCE_ID\"" opts=$opts" --instance-id=\"$INSTANCE_ID\""
@ -3378,9 +3414,9 @@ opts="${opts# *}"
GetLocalOS GetLocalOS
InitLocalOSSettings InitLocalOSSettings
CheckEnvironment
PreInit PreInit
Init Init
CheckEnvironment
PostInit PostInit
if [ $_QUICK_SYNC -lt 2 ]; then if [ $_QUICK_SYNC -lt 2 ]; then
CheckCurrentConfig CheckCurrentConfig