WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2017-07-27 23:20:24 +02:00
parent f26b1118a1
commit 86ff4dc5e9
5 changed files with 5296 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
dev/debug_osync.sh
View File

@ -2,17 +2,13 @@
#TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ? #TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ?
#TODO check if _getCtimeMtime | sort removal needs to be backported #TODO check if _getCtimeMtime | sort removal needs to be backported
#TODO backport treeList sed -r sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' fix && _getFileCtimeMtime* IFS read fix #Check dryruns with nosuffix mode for timestampList
#TODO LANG=C... backport to v1.2.1 and v1.1
#TODO: conflict list is not mandatory, but is still needed for acl resolution
#TODO: syncAttrs must move the file list to sub function, which checks which kind of file list to use
#TODO: double .xz extension when sending email alert with attachment
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong" AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.2-dev PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017060801 PROGRAM_BUILD=2017072701
IS_STABLE=no IS_STABLE=no
@ -48,7 +44,7 @@ IS_STABLE=no
_OFUNCTIONS_VERSION=2.1.4-rc1 _OFUNCTIONS_VERSION=2.1.4-rc1
_OFUNCTIONS_BUILD=2017060701 _OFUNCTIONS_BUILD=2017060903
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -441,7 +437,6 @@ function SendAlert {
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=false attachment=false
else else
attachment=true attachment=true
@ -1738,7 +1733,10 @@ function SetCompression {
COMPRESSION_EXTENSION= COMPRESSION_EXTENSION=
fi fi
fi fi
if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
fi
} }
function InitLocalOSDependingSettings { function InitLocalOSDependingSettings {
@ -2935,7 +2933,7 @@ function treeList {
# operation explanation # operation explanation
# (command || :) = Return code 0 regardless of command return code # (command || :) = Return code 0 regardless of command return code
# (grep -E \"^-|^d|^l\" || :) = Be sure line begins with '-' or 'd' or 'l' (rsync semantics for file, directory or symlink) # (grep -E \"^-|^d|^l\" || :) = Be sure line begins with '-' or 'd' or 'l' (rsync semantics for file, directory or symlink)
# (sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) = Remove everything before timestamps # (sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) = Remove everything before timestamps
# (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) = Only show output before ' -> ' in order to remove symlink destinations # (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) = Only show output before ' -> ' in order to remove symlink destinations
# (grep -v \"^\.$\" || :) = Removes line containing current directory sign '.' # (grep -v \"^\.$\" || :) = Removes line containing current directory sign '.'
@ -2943,9 +2941,9 @@ function treeList {
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
else else
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
fi fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
@ -3102,7 +3100,7 @@ ENDSSH
sed -i.tmp -e 's/^\\"//' -e 's/\\"$//' "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" sed -i.tmp -e 's/^\\"//' -e 's/\\"$//' "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP"
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Cannot fix FreeBDS 11 remote" "ERROR" Logger "Cannot fix FreeBSD 11 remote csh syntax." "ERROR"
return $retval return $retval
fi fi
cat "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" | sort > "$timestampFile" cat "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" | sort > "$timestampFile"
@ -4598,6 +4596,7 @@ function Init {
readonly __timestampAfterFile=17 readonly __timestampAfterFile=17
readonly __timestampAfterFileNoSuffix=18 readonly __timestampAfterFileNoSuffix=18
readonly __conflictListFile=19 readonly __conflictListFile=19
readonly __updateTriggerFile=20
INITIATOR=() INITIATOR=()
INITIATOR[$__type]='initiator' INITIATOR[$__type]='initiator'
@ -4620,6 +4619,7 @@ function Init {
INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix" INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID" INITIATOR[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix" INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
INITIATOR[$__updateTriggerFile]="$INITIATOR_SYNC_DIR$OSYNC_DIR/.osync-update.push"
TARGET=() TARGET=()
TARGET[$__type]='target' TARGET[$__type]='target'
@ -4634,7 +4634,6 @@ function Init {
TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused
#WIP NoSuffix file to add to timestamp
TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused
TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused
TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
@ -4643,6 +4642,7 @@ function Init {
TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix" TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
TARGET[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID" TARGET[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix" TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
TARGET[$__updateTriggerFile]="$TARGET_SYNC_DIR$OSYNC_DIR/.osync-update.push"
PARTIAL_DIR="${INITIATOR[$__partialDir]}" PARTIAL_DIR="${INITIATOR[$__partialDir]}"
@ -4705,10 +4705,10 @@ function Usage {
echo "$AUTHOR" echo "$AUTHOR"
echo "$CONTACT" echo "$CONTACT"
echo "" echo ""
echo "You may use osync with a full blown configuration file, or use its default options for quick command line sync." echo "You may use $PROGRAM with a full blown configuration file, or use its default options for quick command line sync."
echo "Usage: osync.sh /path/to/config/file [OPTIONS]" echo "Usage: $0 /path/to/config/file [OPTIONS]"
echo "or osync.sh --initiator=/path/to/initiator/replica --target=/path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]" echo "or $0 --initiator=/path/to/initiator/replica --target=/path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync.sh --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]" echo "or $0 --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "" echo ""
echo "[OPTIONS]" echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing" echo "--dry Will run osync without actually doing anything; just testing"
@ -4738,7 +4738,7 @@ function Usage {
echo "--destination-mails=\"\" Double quoted list of space separated email addresses to send alerts to" echo "--destination-mails=\"\" Double quoted list of space separated email addresses to send alerts to"
echo "" echo ""
echo "Additionaly, you may set most osync options at runtime. eg:" echo "Additionaly, you may set most osync options at runtime. eg:"
echo "SOFT_DELETE_DAYS=365 osync.sh --initiator=/path --target=/other/path" echo "SOFT_DELETE_DAYS=365 $0 --initiator=/path --target=/other/path"
echo "" echo ""
exit 128 exit 128
} }
@ -4766,7 +4766,7 @@ function SyncOnChanges {
exit 1 exit 1
fi fi
Logger "#### Running osync in file monitor mode." "NOTICE" Logger "#### Running $PROGRAM in file monitor mode." "NOTICE"
while true; do while true; do
if [ "$ConfigFile" != "" ]; then if [ "$ConfigFile" != "" ]; then
@ -4835,12 +4835,15 @@ _NOLOCKS=false
osync_cmd=$0 osync_cmd=$0
_SUMMARY=false _SUMMARY=false
function GetCommandlineArguments {
local isFirstArgument=true
if [ $# -eq 0 ] if [ $# -eq 0 ]
then then
Usage Usage
fi fi
first=1
for i in "$@"; do for i in "$@"; do
case $i in case $i in
--dry) --dry)
@ -4940,27 +4943,28 @@ for i in "$@"; do
_REMOTE_TOKEN=${i##*=} _REMOTE_TOKEN=${i##*=}
;; ;;
*) *)
if [ $first == "0" ]; then if [ $isFirstArgument == false ]; then
Logger "Unknown option '$i'" "CRITICAL" Logger "Unknown option '$i'" "CRITICAL"
Usage Usage
fi fi
;; ;;
esac esac
first=0 isFirstArgument=false
done done
# Remove leading space if there is one # Remove leading space if there is one
opts="${opts# *}" opts="${opts# *}"
}
GetCommandlineArguments "$@"
## Here we set default options for quicksync tasks when no configuration file is provided. ## Here we set default options for quicksync tasks when no configuration file is provided.
if [ $_QUICK_SYNC -eq 2 ]; then if [ $_QUICK_SYNC -eq 2 ]; then
if [ "$INSTANCE_ID" == "" ]; then if [ "$INSTANCE_ID" == "" ]; then
INSTANCE_ID="quicksync_task" INSTANCE_ID="quicksync_task"
fi fi
# Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh # Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh
if [ $(IsInteger $MINIMUM_SPACE) -ne 1 ]; then if [ $(IsInteger $MINIMUM_SPACE) -ne 1 ]; then
MINIMUM_SPACE=1024 MINIMUM_SPACE=1024
fi fi
@ -4980,6 +4984,7 @@ opts="${opts# *}"
if [ $(IsInteger $SOFT_MAX_EXEC_TIME) -ne 1 ]; then if [ $(IsInteger $SOFT_MAX_EXEC_TIME) -ne 1 ]; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
fi fi
if [ $(IsInteger $HARD_MAX_EXEC_TIME) -ne 1 ]; then if [ $(IsInteger $HARD_MAX_EXEC_TIME) -ne 1 ]; then
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
@ -4987,7 +4992,6 @@ opts="${opts# *}"
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";" PATH_SEPARATOR_CHAR=";"
fi fi
MIN_WAIT=30 MIN_WAIT=30
else else
ConfigFile="${1}" ConfigFile="${1}"

2520
dev/debug_osync_target_helper.sh Executable file
View File

File diff suppressed because it is too large Load Diff

70
install.sh
View File

@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh" SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2017041701 SCRIPT_BUILD=2017072701
## osync / obackup / pmocr / zsnap install script ## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@ -21,6 +21,39 @@ SCRIPT_BUILD=2017041701
# Get current install.sh path from http://stackoverflow.com/a/246128/2635443 # Get current install.sh path from http://stackoverflow.com/a/246128/2635443
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
_LOGGER_SILENT=false
_STATS=1
ACTION="install"
FAKEROOT=""
function GetCommandlineArguments {
for i in "$@"; do
case $i in
--prefix=*)
FAKEROOT="${i##*=}"
;;
--silent)
_LOGGER_SILENT=true
;;
--no-stats)
_STATS=0
;;
--remove)
ACTION="uninstall"
;;
--help|-h|-?)
Usage
;;
*)
Logger "Unknown option '$i'" "CRITICAL"
Usage
;;
esac
done
}
GetCommandlineArguments "$@"
CONF_DIR=$FAKEROOT/etc/$PROGRAM CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin" BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
@ -227,7 +260,7 @@ function CreateDir {
local dir="${1}" local dir="${1}"
if [ ! -d "$dir" ]; then if [ ! -d "$dir" ]; then
mkdir "$dir" mkdir -p "$dir"
if [ $? == 0 ]; then if [ $? == 0 ]; then
QuickLogger "Created directory [$dir]." QuickLogger "Created directory [$dir]."
else else
@ -331,8 +364,10 @@ function CopyProgram {
function CopyServiceFiles { function CopyServiceFiles {
if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
CreateDir "$SERVICE_DIR_SYSTEMD_SYSTEM"
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_USER" ]; then
CreateDir "$SERVICE_DIR_SYSTEMD_USER"
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
fi fi
@ -341,6 +376,7 @@ function CopyServiceFiles {
QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]."
QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]." QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]."
elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
CreateDir "$SERVICE_DIR_INIT"
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
@ -398,7 +434,7 @@ function RemoveAll {
QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it." QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it."
fi fi
RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM"
RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM" RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER"
RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT" RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT"
QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually." QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually."
@ -410,34 +446,10 @@ function Usage {
echo "--silent Will log and bypass user interaction." echo "--silent Will log and bypass user interaction."
echo "--no-stats Used with --silent in order to refuse sending anonymous install stats." echo "--no-stats Used with --silent in order to refuse sending anonymous install stats."
echo "--remove Remove the program." echo "--remove Remove the program."
echo "--prefix=/path Use prefix to install path."
exit 127 exit 127
} }
_LOGGER_SILENT=false
_STATS=1
ACTION="install"
for i in "$@"
do
case $i in
--silent)
_LOGGER_SILENT=true
;;
--no-stats)
_STATS=0
;;
--remove)
ACTION="uninstall"
;;
--help|-h|-?)
Usage
esac
done
if [ "$FAKEROOT" != "" ]; then
mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
fi
GetLocalOS GetLocalOS
SetLocalOSSettings SetLocalOSSettings
GetInit GetInit

54
osync.sh
View File

@ -2,24 +2,20 @@
#TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ? #TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ?
#TODO check if _getCtimeMtime | sort removal needs to be backported #TODO check if _getCtimeMtime | sort removal needs to be backported
#TODO backport treeList sed -r sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' fix && _getFileCtimeMtime* IFS read fix #Check dryruns with nosuffix mode for timestampList
#TODO LANG=C... backport to v1.2.1 and v1.1
#TODO: conflict list is not mandatory, but is still needed for acl resolution
#TODO: syncAttrs must move the file list to sub function, which checks which kind of file list to use
#TODO: double .xz extension when sending email alert with attachment
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong" AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.2-dev PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017060801 PROGRAM_BUILD=2017072701
IS_STABLE=no IS_STABLE=no
_OFUNCTIONS_VERSION=2.1.4-rc1 _OFUNCTIONS_VERSION=2.1.4-rc1
_OFUNCTIONS_BUILD=2017060701 _OFUNCTIONS_BUILD=2017060903
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -394,7 +390,6 @@ function SendAlert {
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=false attachment=false
else else
attachment=true attachment=true
@ -1582,7 +1577,10 @@ function SetCompression {
COMPRESSION_EXTENSION= COMPRESSION_EXTENSION=
fi fi
fi fi
if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
fi
} }
function InitLocalOSDependingSettings { function InitLocalOSDependingSettings {
@ -2746,7 +2744,7 @@ function treeList {
# operation explanation # operation explanation
# (command || :) = Return code 0 regardless of command return code # (command || :) = Return code 0 regardless of command return code
# (grep -E \"^-|^d|^l\" || :) = Be sure line begins with '-' or 'd' or 'l' (rsync semantics for file, directory or symlink) # (grep -E \"^-|^d|^l\" || :) = Be sure line begins with '-' or 'd' or 'l' (rsync semantics for file, directory or symlink)
# (sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) = Remove everything before timestamps # (sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) = Remove everything before timestamps
# (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) = Only show output before ' -> ' in order to remove symlink destinations # (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) = Only show output before ' -> ' in order to remove symlink destinations
# (grep -v \"^\.$\" || :) = Removes line containing current directory sign '.' # (grep -v \"^\.$\" || :) = Removes line containing current directory sign '.'
@ -2754,9 +2752,9 @@ function treeList {
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
else else
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -E 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
fi fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
@ -2910,7 +2908,7 @@ ENDSSH
sed -i.tmp -e 's/^\\"//' -e 's/\\"$//' "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" sed -i.tmp -e 's/^\\"//' -e 's/\\"$//' "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP"
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Cannot fix FreeBDS 11 remote" "ERROR" Logger "Cannot fix FreeBSD 11 remote csh syntax." "ERROR"
return $retval return $retval
fi fi
cat "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" | sort > "$timestampFile" cat "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" | sort > "$timestampFile"
@ -4382,6 +4380,7 @@ function Init {
readonly __timestampAfterFile=17 readonly __timestampAfterFile=17
readonly __timestampAfterFileNoSuffix=18 readonly __timestampAfterFileNoSuffix=18
readonly __conflictListFile=19 readonly __conflictListFile=19
readonly __updateTriggerFile=20
INITIATOR=() INITIATOR=()
INITIATOR[$__type]='initiator' INITIATOR[$__type]='initiator'
@ -4404,6 +4403,7 @@ function Init {
INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix" INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID" INITIATOR[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix" INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
INITIATOR[$__updateTriggerFile]="$INITIATOR_SYNC_DIR$OSYNC_DIR/.osync-update.push"
TARGET=() TARGET=()
TARGET[$__type]='target' TARGET[$__type]='target'
@ -4418,7 +4418,6 @@ function Init {
TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused
#WIP NoSuffix file to add to timestamp
TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused
TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused
TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix" TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
@ -4427,6 +4426,7 @@ function Init {
TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix" TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
TARGET[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID" TARGET[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix" TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
TARGET[$__updateTriggerFile]="$TARGET_SYNC_DIR$OSYNC_DIR/.osync-update.push"
PARTIAL_DIR="${INITIATOR[$__partialDir]}" PARTIAL_DIR="${INITIATOR[$__partialDir]}"
@ -4487,10 +4487,10 @@ function Usage {
echo "$AUTHOR" echo "$AUTHOR"
echo "$CONTACT" echo "$CONTACT"
echo "" echo ""
echo "You may use osync with a full blown configuration file, or use its default options for quick command line sync." echo "You may use $PROGRAM with a full blown configuration file, or use its default options for quick command line sync."
echo "Usage: osync.sh /path/to/config/file [OPTIONS]" echo "Usage: $0 /path/to/config/file [OPTIONS]"
echo "or osync.sh --initiator=/path/to/initiator/replica --target=/path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]" echo "or $0 --initiator=/path/to/initiator/replica --target=/path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync.sh --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]" echo "or $0 --initiator=/path/to/initiator/replica --target=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/target/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "" echo ""
echo "[OPTIONS]" echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing" echo "--dry Will run osync without actually doing anything; just testing"
@ -4520,7 +4520,7 @@ function Usage {
echo "--destination-mails=\"\" Double quoted list of space separated email addresses to send alerts to" echo "--destination-mails=\"\" Double quoted list of space separated email addresses to send alerts to"
echo "" echo ""
echo "Additionaly, you may set most osync options at runtime. eg:" echo "Additionaly, you may set most osync options at runtime. eg:"
echo "SOFT_DELETE_DAYS=365 osync.sh --initiator=/path --target=/other/path" echo "SOFT_DELETE_DAYS=365 $0 --initiator=/path --target=/other/path"
echo "" echo ""
exit 128 exit 128
} }
@ -4547,7 +4547,7 @@ function SyncOnChanges {
exit 1 exit 1
fi fi
Logger "#### Running osync in file monitor mode." "NOTICE" Logger "#### Running $PROGRAM in file monitor mode." "NOTICE"
while true; do while true; do
if [ "$ConfigFile" != "" ]; then if [ "$ConfigFile" != "" ]; then
@ -4616,12 +4616,15 @@ _NOLOCKS=false
osync_cmd=$0 osync_cmd=$0
_SUMMARY=false _SUMMARY=false
function GetCommandlineArguments {
local isFirstArgument=true
if [ $# -eq 0 ] if [ $# -eq 0 ]
then then
Usage Usage
fi fi
first=1
for i in "$@"; do for i in "$@"; do
case $i in case $i in
--dry) --dry)
@ -4721,27 +4724,28 @@ for i in "$@"; do
_REMOTE_TOKEN=${i##*=} _REMOTE_TOKEN=${i##*=}
;; ;;
*) *)
if [ $first == "0" ]; then if [ $isFirstArgument == false ]; then
Logger "Unknown option '$i'" "CRITICAL" Logger "Unknown option '$i'" "CRITICAL"
Usage Usage
fi fi
;; ;;
esac esac
first=0 isFirstArgument=false
done done
# Remove leading space if there is one # Remove leading space if there is one
opts="${opts# *}" opts="${opts# *}"
}
GetCommandlineArguments "$@"
## Here we set default options for quicksync tasks when no configuration file is provided. ## Here we set default options for quicksync tasks when no configuration file is provided.
if [ $_QUICK_SYNC -eq 2 ]; then if [ $_QUICK_SYNC -eq 2 ]; then
if [ "$INSTANCE_ID" == "" ]; then if [ "$INSTANCE_ID" == "" ]; then
INSTANCE_ID="quicksync_task" INSTANCE_ID="quicksync_task"
fi fi
# Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh # Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh
if [ $(IsInteger $MINIMUM_SPACE) -ne 1 ]; then if [ $(IsInteger $MINIMUM_SPACE) -ne 1 ]; then
MINIMUM_SPACE=1024 MINIMUM_SPACE=1024
fi fi
@ -4761,6 +4765,7 @@ opts="${opts# *}"
if [ $(IsInteger $SOFT_MAX_EXEC_TIME) -ne 1 ]; then if [ $(IsInteger $SOFT_MAX_EXEC_TIME) -ne 1 ]; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
fi fi
if [ $(IsInteger $HARD_MAX_EXEC_TIME) -ne 1 ]; then if [ $(IsInteger $HARD_MAX_EXEC_TIME) -ne 1 ]; then
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
@ -4768,7 +4773,6 @@ opts="${opts# *}"
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";" PATH_SEPARATOR_CHAR=";"
fi fi
MIN_WAIT=30 MIN_WAIT=30
else else
ConfigFile="${1}" ConfigFile="${1}"

2375
osync_target_helper.sh Executable file
View File

File diff suppressed because it is too large Load Diff