Compare commits

...

103 Commits

Author SHA1 Message Date
Orsiris de Jong 8a3478e363 Added FreeBSD 11 note 2017-06-09 23:03:01 +02:00
deajan 0e6b3b80b5 Added msys2 to supported OS list 2017-06-09 14:54:52 +02:00
deajan 679180eaef Fixed CleanDebug on FreeBSD 2017-06-09 14:48:34 +02:00
deajan 9cc7edbf27 Changed sed -E to sed -r for compat issues 2017-06-08 16:33:59 +02:00
deajan aac8b6c97b Fixed new regex for rsync output 2017-06-06 17:11:20 +02:00
deajan e82e1488ee Bumped to stable 2017-06-06 16:05:26 +02:00
deajan f448a15733 Update changelog 2017-06-06 16:04:49 +02:00
deajan 6d2eb4769b Testing escaped remote IFS 2017-06-06 10:01:32 +02:00
deajan 4b25c60b87 Fixed escaped singlequotes 2017-06-06 09:58:20 +02:00
deajan 8efec3947c Backported ACL fixes 2017-06-06 09:51:20 +02:00
deajan 60c6657ba8 Rebuilt targets 2017-05-31 18:17:03 +02:00
deajan 063eca84e4 Backported fixes from v1.2x 2017-05-31 18:16:13 +02:00
deajan 4de0a657c2 Added directory soft deletion tests 2016-12-13 01:59:25 +01:00
deajan 81571c605d Updated unit tests from 1.2 2016-12-10 11:55:53 +01:00
deajan d00485e9db Updated unit tests from v1.2 2016-12-04 18:59:48 +01:00
deajan aab1940520 Removed mail sending from travis tests 2016-11-30 13:49:36 +01:00
deajan 5ef5297ac5 Reenabled all tests again 2016-11-30 12:59:52 +01:00
deajan 3f6e707db4 Updated changelog 2016-11-30 12:56:56 +01:00
deajan 92f95e309a Added timed execution tests for v1.1 branch 2016-11-30 12:56:25 +01:00
deajan fd204349fb Improved timed behavior 2016-11-30 12:55:48 +01:00
deajan bf8cacd4b6 Updated travis file with sudo and dependancies 2016-11-22 13:11:48 +01:00
deajan 130acc2cba Better unit tests 2016-11-22 10:07:51 +01:00
deajan fe6418ba17 Added more ACL tests 2016-11-20 17:44:42 +01:00
deajan bc301c6c00 Updated tests from v1.2 2016-11-18 00:11:22 +01:00
deajan e0a100827f Keep local 3rdParty remote hosts 2016-11-17 17:19:20 +01:00
deajan 902aaa8e83 Updated travis local.conf file 2016-11-17 17:17:24 +01:00
Orsiris de Jong 2e57f7bc86 Update README.md 2016-11-17 14:12:04 +01:00
deajan 2da38fd0fa Rebuilt targets 2016-11-17 13:43:03 +01:00
deajan 432919b3ae Fixed conf files for tests 2016-11-17 13:11:12 +01:00
deajan f1e1883609 Reenabled all tests :) 2016-11-17 12:43:18 +01:00
deajan 0b17fd3302 Updated changelog 2016-11-17 12:41:04 +01:00
deajan a284528ca6 Updated unit tests from v1.2 2016-11-17 12:33:37 +01:00
deajan a485ec25c5 Fixed remote locking always resumed 2016-11-17 12:33:13 +01:00
deajan 77ec256bdb Removed earlier non used test folder 2016-11-16 20:03:33 +01:00
deajan aeff4b8261 Merge branch 'v1.1-maint' of https://github.com/deajan/osync into v1.1-maint 2016-11-16 00:48:58 +01:00
deajan 044d83907b Fixed tests depending on minor version 2016-11-16 00:48:07 +01:00
deajan d3643d0e9b Local runs should not test connectivity 2016-11-16 00:45:01 +01:00
Orsiris de Jong 9445f5532e Added travis building status 2016-11-15 23:23:09 +01:00
Orsiris de Jong cb8eb9c2e1 Update README.md 2016-11-15 23:22:53 +01:00
deajan 1ceb7901aa Added unit tests 2016-11-15 23:21:04 +01:00
deajan 62cce235c0 Updated travis file for new unit tests 2016-11-15 22:50:07 +01:00
deajan e3894d3a9d Backport RsyncPattern function from v1.2 2016-11-15 22:39:16 +01:00
deajan b66b9669fa Always launch RsyncPatterns function 2016-11-15 22:35:58 +01:00
deajan 5af05c2878 Added default separator char in quicksync mode 2016-11-15 22:34:08 +01:00
deajan 4ca9689471 Add possibility to add rsync exclusions on quicksync mode 2016-11-15 22:21:14 +01:00
deajan fbc6b5d3c2 Fixed logger flags not gotten from subprocesses 2016-11-15 22:09:47 +01:00
deajan 1ba222ba89 Added logger flag files 2016-11-15 22:09:28 +01:00
deajan 9c6439e243 Fixed backed up file path 2016-11-15 21:55:54 +01:00
deajan 0b2359d0ca Fixed bash 4.2 compatibility 2016-11-12 14:05:02 +01:00
deajan 1278b420d0 Better way to get current version of stable 2016-11-12 13:58:44 +01:00
deajan dd70b90f98 Added Shadowigor's AUR package script from PR#70 2016-11-12 13:38:16 +01:00
deajan 58282b5fc0 Rebuilt targets 2016-11-10 16:29:39 +01:00
deajan 5f5ac1850d Fixed logging issue with multiple users 2016-11-10 16:29:20 +01:00
deajan 132ae6ec8b Make travis happy 2016-09-02 21:57:04 +02:00
deajan 36f85ce6db Added basic travis file 2016-09-02 21:52:29 +02:00
deajan 9e2f07153c Updated changelog 2016-09-02 21:47:06 +02:00
deajan 21e98e474c Rebuilt targets 2016-09-02 21:46:11 +02:00
deajan 2b14648e14 Fixed EscapeSpaces with bash >= 4.3 2016-09-02 21:45:37 +02:00
Orsiris de Jong 1f5a510b6e Fixed os detection (again after PR62) 2016-08-29 20:05:42 +02:00
Orsiris de Jong e4acfc029d Re introduced os detection fix that disappeared with PR2 2016-08-29 20:03:32 +02:00
Orsiris de Jong 44c341a7af Merge pull request #63 from deajan/stable
Merged PR60 / PR62
2016-08-29 20:02:32 +02:00
Orsiris de Jong 528d35c64e Merge pull request #62 from Shadowigor/stable-fakeroot
Add fakeroot support for Arch packaging in stable branch
2016-08-29 20:00:33 +02:00
Shadowigor 1ddd051033 Added fakeroot support for Arch packaging 2016-08-29 19:38:04 +02:00
deajan cd2b4b6154 Updated changelog 2016-08-29 17:51:55 +02:00
deajan e5813e6741 Fixed os detection 2016-08-29 17:41:34 +02:00
deajan eab6a05e4f Updated readme for new release 2016-08-28 11:50:18 +02:00
deajan 250d8a4b18 Renamed sync.conf to sync.conf.example 2016-08-28 11:39:49 +02:00
deajan ea0a1770db mend
Updated Changelog
2016-08-28 11:39:16 +02:00
deajan 0e37289fdc Updated changeelog 2016-08-28 11:39:06 +02:00
deajan 66c2735564 Rebuilt targets 2016-08-22 10:30:14 +02:00
deajan ef3153cbcd Fixed RunAfterHook double execution 2016-08-22 10:29:56 +02:00
deajan f70d8cda6a Rebuilt target 2016-08-13 12:14:35 +02:00
deajan 04585c720b Updated changelog 2016-08-13 12:12:22 +02:00
deajan 05975c3fa5 Rebuilt targets 2016-08-13 12:04:08 +02:00
deajan 3fb6bbb98c Added common installer template 2016-08-13 12:03:51 +02:00
deajan 4a60e64ca1 Fixed soft deletion when SUDO_EXEC enabled 2016-08-13 12:02:50 +02:00
Orsiris de Jong c906f001a4 Update README.md 2016-08-09 15:41:48 +02:00
deajan 665bc67af6 Updated install details 2016-08-06 14:07:31 +02:00
deajan 57b32d28fa Rebuilt targets 2016-08-06 13:27:37 +02:00
deajan 9bc8232071 Fixed double pattern declaration 2016-08-06 13:27:01 +02:00
deajan 3bd39e19c6 Updated changelog 2016-08-02 22:13:10 +02:00
deajan e8779625ce Rebuild targets 2016-08-02 22:12:57 +02:00
deajan 008b74200d Fixed soft deletion always enabled remotely 2016-08-02 22:12:03 +02:00
deajan fcfb26b19b Rebuild targets 2016-08-02 13:00:03 +02:00
deajan a76df153f0 Updated changelog 2016-08-02 12:59:48 +02:00
deajan 108bf7d316 Fixed attribute list for ctimes 2016-08-02 12:59:33 +02:00
deajan fc31fe4f5f Target rebuilds 2016-08-02 10:06:25 +02:00
deajan 8c10e2f8ef Fixed verbose warning for missing deletion dirs 2016-08-02 10:04:00 +02:00
deajan 8e3b5bf762 Revert "Fix for missing state subdirs"
This reverts commit 021b354929.
2016-08-02 10:02:44 +02:00
deajan 2abe5c36c1 Revert "missing state subdirs"
This reverts commit 2de50b413c.
2016-08-02 10:02:09 +02:00
deajan 2de50b413c Target rebuild 2016-08-01 14:18:41 +02:00
deajan 021b354929 Fix for missing state subdirs 2016-08-01 14:18:09 +02:00
deajan 0d7aa043a2 Rebuild targets 2016-08-01 13:49:00 +02:00
deajan 010e171f50 More remote deletion love 2016-08-01 13:34:17 +02:00
deajan 9b01a7533f Prevent remote deletion to fail on time mismatch between systems 2016-08-01 13:18:21 +02:00
deajan bdeb7fe907 Re-rebuild for v1.1.1 2016-08-01 12:48:01 +02:00
deajan 2d3e945817 Rebuilt osync for v1.1.1 2016-08-01 12:20:37 +02:00
deajan c234bdf107 More fixes for remote deletion from PR55 2016-08-01 12:19:48 +02:00
deajan c209fdd116 Fixes for remote deletion from PR55 2016-08-01 12:16:16 +02:00
deajan 6196ce6b31 More misc fixes 2016-08-01 11:03:49 +02:00
deajan 13537c5768 Fixed doubled path 2016-08-01 10:48:16 +02:00
deajan 05d2a33b7e Fix typo 2016-08-01 10:46:04 +02:00
deajan 98024c824e Misc fix 2016-08-01 10:41:50 +02:00
26 changed files with 3391 additions and 1303 deletions

16
.travis.yml Normal file
View File

@ -0,0 +1,16 @@
language:
bash
os:
linux
osx
before_install:
sudo apt-get install inotify-tools acl
sudo:
required
script:
TRAVIS_RUN=true dev/tests/run_tests.sh

View File

@ -7,6 +7,44 @@ KNOWN ISSUES
RECENT CHANGES RECENT CHANGES
-------------- --------------
dd Mmm YYYY: osync v1.1.6 released
- Backported v1.2.1 fixes
- Fixed an issue with filenames ending with spaces, their deletion not being propagated, and ACL / conflicts not being managed (still they got synced)
- Fixed bogus pgrep can lead to segfault 11 because of recursive KillChilds
- Fixed osync deletion not working on systems with ssh banner enabled
- Fixed low severity security issue where log and run files could be read by other users
- SLEEP_TIME, SOFT_MAX_EXEC_TIME and HARD_MAX_EXEC_TIME can now be set as environment variables
- Backported unit tests from v1.2-beta allowing to fix the following
- HARD_MAX_EXEC_TIME wasn't enforced properly
17 Nov 2016: osync v1.1.5 released
- Backported unit tests from v1.2-beta allowing to fix the following
- Allow quicksync mode to specify rsync include / exclude patterns as environment variables
- Added default path separator char in quicksync mode for multiple includes / exclusions
- Local runs should not check for remote connectivity
- Fixed backups go into root of replica instead of .osync_wordir/backups
- Fixed error alerts cannot be triggered from subprocesses
- Fixed remote locked targets are unlocked in any case
10 Nov 2016: osync v1.1.4 released
- Fixed a corner case with sending alerts with logfile attachments when osync is used by multiple users
02 Sep 2016: osync v1.1.3 released
- Fixed directories containing spaces with bash >= 4.3
- Fixed installer for CYGWIN / MSYS environment
28 Aug 2016: osync v1.1.2 released
- Renamed sync.conf to sync.conf.example (thanks to https://github.com/hortimech)
- Fixed RunAfterHook may be executed twice
- Fixed soft deletion when SUDO_EXEC is enabled
06 Aug 2016: osync v1.1.1 released
- Fixed bogus rsync pattern additions
- Fixed soft deletion always enabled on target
- Fixed problem with attributes file list function
- Fixed deletion propagation code
- Fixed missing deletion / backup diretories message in verbose mode
27 Jul 2016: osync v1.1 released 27 Jul 2016: osync v1.1 released
- More msys and cygwin compatibility - More msys and cygwin compatibility
- Logging begins now before any remote checks - Logging begins now before any remote checks

View File

@ -1,5 +1,4 @@
osync # osync [![Build Status](https://travis-ci.org/deajan/osync.svg?branch=v1.1-maint)](https://travis-ci.org/deajan/osync) [![GitHub Release](https://img.shields.io/github/release/deajan/osync.svg?label=Latest)](https://github.com/deajan/osync/releases/latest)
=====
A two way filesync script with fault tolerance, resume, soft deletion, conflictual file backups running on bash (linux, BSD and virtually any system supporting bash). A two way filesync script with fault tolerance, resume, soft deletion, conflictual file backups running on bash (linux, BSD and virtually any system supporting bash).
File synchronization is bidirectional, based on rsync, can be run manually, as scheduled task, or triggered on file changes. File synchronization is bidirectional, based on rsync, can be run manually, as scheduled task, or triggered on file changes.
@ -26,6 +25,7 @@ osync tasks may be launched sequentially by osync osync-batch tool.
Currently, it has been tested on CentOS 5.x, 6.x, 7.x, Debian 6, Debian 7, Linux Mint 14-17, Ubuntu 12.04, 12.10, FreeBSD 8.3, 10.1, 10.3, Mac OS X and pfSense. Currently, it has been tested on CentOS 5.x, 6.x, 7.x, Debian 6, Debian 7, Linux Mint 14-17, Ubuntu 12.04, 12.10, FreeBSD 8.3, 10.1, 10.3, Mac OS X and pfSense.
Microsoft Windows is supported via MSYS or Cygwin. Microsoft Windows is supported via MSYS or Cygwin.
Note that FreeBSD 11 is only supported in versions v1.2+.
## Installation ## Installation
@ -33,11 +33,12 @@ Microsoft Windows is supported via MSYS or Cygwin.
Osync has been designed to not delete any data, but rather make backups of conflictual files or soft deletes. Osync has been designed to not delete any data, but rather make backups of conflictual files or soft deletes.
Nevertheless, you should always have a neat backup of your data before trying a new sync tool. Nevertheless, you should always have a neat backup of your data before trying a new sync tool.
You can download the latest stable release of Osync at www.netpower.fr/osync or https://github.com/deajan/osync/archive/v1.1.tar.gz You can download the latest stable release of osync at https://github.com/deajan/osync/archive/stable.tar.gz
You may also get the last development version at https://github.com/deajan/osync with the following command You may also get the last development version at https://github.com/deajan/osync with the following command
$ git clone -b "v1.1" https://github.com/deajan/osync $ git clone https://github.com/deajan/osync
$ cd osync
$ sh install.sh $ sh install.sh
Osync will install itself to /usr/local/bin and an example configuration file will be installed to /etc/osync Osync will install itself to /usr/local/bin and an example configuration file will be installed to /etc/osync

315
dev/common_install.sh Executable file
View File

@ -0,0 +1,315 @@
#!/usr/bin/env bash
PROGRAM=[prgname]
PROGRAM_VERSION=[version]
PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016082902
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
## Please adapt this to fit your distro needs
#TODO: silent mode and no stats mode
CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
## osync specific code
OSYNC_SERVICE_FILE_INIT="osync-srv"
OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM="osync-srv@.service"
OSYNC_SERVICE_FILE_SYSTEMD_USER="osync-srv@.service.user"
## pmocr specfic code
PMOCR_SERVICE_FILE_INIT="pmocr-srv"
PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM="pmocr-srv.service"
## Generic code
## Default log file
if [ -w $FAKEROOT/var/log ]; then
LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM-install.log"
else
LOG_FILE="./$PROGRAM-install.log"
fi
# Generic quick logging function
function _QuickLogger {
local value="${1}"
local destination="${2}" # Destination: stdout, log, both
if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
echo -e "$(date) - $value" >> "$LOG_FILE"
elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
echo -e "$value"
fi
}
function QuickLogger {
local value="${1}"
if [ "$_SILENT" -eq 1 ]; then
_QuickLogger "$value" "log"
else
_QuickLogger "$value" "stdout"
fi
}
function urlencode() {
# urlencode <string>
local LANG=C
local length="${#1}"
for (( i = 0; i < length; i++ )); do
local c="${1:i:1}"
case $c in
[a-zA-Z0-9.~_-]) printf "$c" ;;
*) printf '%%%02X' "'$c" ;;
esac
done
}
function SetOSSettings {
USER=root
local local_os_var
local_os_var="$(uname -spio 2>&1)"
if [ $? != 0 ]; then
local_os_var="$(uname -v 2>&1)"
if [ $? != 0 ]; then
local_os_var="$(uname)"
fi
fi
case $local_os_var in
*"BSD"*)
GROUP=wheel
;;
*"Darwin"*)
GROUP=admin
;;
*"MINGW32"*|*"CYGWIN"*)
USER=""
GROUP=""
;;
*)
GROUP=root
;;
esac
if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
QuickLogger "Must be run as $USER."
exit 1
fi
OS=$(urlencode "$local_os_var")
}
function GetInit {
if [ -f /sbin/init ]; then
if file /sbin/init | grep systemd > /dev/null; then
init="systemd"
else
init="initV"
fi
else
QuickLogger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron."
init="none"
fi
}
function CreateConfDir {
if [ ! -d "$CONF_DIR" ]; then
mkdir "$CONF_DIR"
if [ $? == 0 ]; then
QuickLogger "Created directory [$CONF_DIR]."
else
QuickLogger "Cannot create directory [$CONF_DIR]."
exit 1
fi
else
QuickLogger "Config directory [$CONF_DIR] exists."
fi
}
function CopyExampleFiles {
if [ -f "./sync.conf.example" ]; then
cp "./sync.conf.example" "$FAKEROOT/etc/$PROGRAM/sync.conf.example"
fi
if [ -f "./host_backup.conf.example" ]; then
cp "./host_backup.conf.example" "$FAKEROOT/etc/$PROGRAM/host_backup.conf.example"
fi
if [ -f "./exlude.list.example" ]; then
cp "./exclude.list.example" "$FAKEROOT/etc/$PROGRAM"
fi
if [ -f "./snapshot.conf.example" ]; then
cp "./snapshot.conf.example" "$FAKEROOT/etc/$PROGRAM/snapshot.conf.example"
fi
}
function CopyProgram {
cp "./$PROGRAM_BINARY" "$BIN_DIR"
if [ $? != 0 ]; then
QuickLogger "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]. Make sure to run install script in the directory containing all other files."
QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
exit 1
else
chmod 755 "$BIN_DIR/$PROGRAM_BINARY"
QuickLogger "Copied $PROGRAM_BINARY to [$BIN_DIR]."
fi
if [ -f "./$PROGRAM_BATCH" ]; then
cp "./$PROGRAM_BATCH" "$BIN_DIR"
if [ $? != 0 ]; then
QuickLogger "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]."
else
chmod 755 "$BIN_DIR/$PROGRAM_BATCH"
QuickLogger "Copied $PROGRAM_BATCH to [$BIN_DIR]."
fi
fi
if [ -f "./ssh_filter.sh" ]; then
cp "./ssh_filter.sh" "$BIN_DIR"
if [ $? != 0 ]; then
QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]."
else
chmod 755 "$BIN_DIR/ssh_filter.sh"
if ([ "$USER" != "" ] && [ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
chown $USER:$GROUP "$BIN_DIR/ssh_filter.sh"
fi
QuickLogger "Copied ssh_filter.sh to [$BIN_DIR]."
fi
fi
}
function CopyServiceFiles {
# OSYNC SPECIFIC
if ([ "$init" == "systemd" ] && [ -f "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
cp "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "./$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
if [ $? != 0 ]; then
QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
else
QuickLogger "Created osync-srv service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
QuickLogger "Can be activated with [systemctl start osync-srv@instance.conf] where instance.conf is the name of the config file in /etc/osync."
QuickLogger "Can be enabled on boot with [systemctl enable osync-srv@instance.conf]."
QuickLogger "In userland, active with [systemctl --user start osync-srv@instance.conf]."
fi
elif ([ "$init" == "initV" ] && [ -f "./$OSYNC_SERVICE_FILE_INIT" ]); then
cp "./$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
if [ $? != 0 ]; then
QuickLogger "Cannot copy osync-srv to [$SERVICE_DIR_INIT]."
else
chmod 755 "$SERVICE_DIR_INIT/$OSYNC_SERVICE_FILE_INIT"
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
fi
fi
# PMOCR SPECIFIC
if ([ "$init" == "systemd" ] && [ -f "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
cp "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM"
if [ $? != 0 ]; then
QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
else
QuickLogger "Created pmocr-srv service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
QuickLogger "Can be activated with [systemctl start pmocr-srv] after configuring file options in [$BIN_DIR/$PROGRAM]."
QuickLogger "Can be enabled on boot with [systemctl enable pmocr-srv]."
fi
elif ([ "$init" == "initV" ] && [ -f "./$PMOCR_SERVICE_FILE_INIT" ]); then
cp "./$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
if [ $? != 0 ]; then
QuickLogger "Cannot copy pmoct-srv to [$SERVICE_DIR_INIT]."
else
chmod 755 "$SERVICE_DIR_INIT/$PMOCR_SERVICE_FILE_INIT"
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $PMOCR_SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $PMOCR_SERVICE_FILE_INIT on]."
fi
fi
}
function Statistics {
if type wget > /dev/null; then
wget -qO- "$STATS_LINK" > /dev/null 2>&1
if [ $? == 0 ]; then
return 0
fi
fi
if type curl > /dev/null; then
curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
if [ $? == 0 ]; then
return 0
fi
fi
QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
return 1
}
function Usage {
echo "Installs $PROGRAM into $BIN_DIR"
echo "options:"
echo "--silent Will log and bypass user interaction."
echo "--no-stats Used with --silent in order to refuse sending anonymous install stats."
exit 127
}
_SILENT=0
_STATS=1
for i in "$@"
do
case $i in
--silent)
_SILENT=1
;;
--no-stats)
_STATS=0
;;
--help|-h|-?)
Usage
esac
done
if [ "$FAKEROOT" != "" ]; then
mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
fi
SetOSSettings
CreateConfDir
CopyExampleFiles
CopyProgram
GetInit
CopyServiceFiles
STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS"
QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
if [ $_STATS -eq 1 ]; then
if [ $_SILENT -eq 1 ]; then
Statistics
else
QuickLogger "In order to make install statistics, the script would like to connect to $STATS_LINK"
read -r -p "No data except those in the url will be send. Allow [Y/n]" response
case $response in
[nN])
exit
;;
*)
Statistics
exit $?
;;
esac
fi
fi

View File

@ -3,11 +3,11 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1 PROGRAM_VERSION=1.1.6-beta
PROGRAM_BUILD=2016072701 PROGRAM_BUILD=2016113001
IS_STABLE=yes IS_STABLE=yes
## FUNC_BUILD=2016071902 ## FUNC_BUILD=2016071902-i
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -24,6 +24,9 @@ KEEP_LOGGING=1801
## Correct output of sort command (language agnostic sorting) ## Correct output of sort command (language agnostic sorting)
export LC_ALL=C export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body # Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -47,14 +50,16 @@ fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.1
_VERBOSE=0 _VERBOSE=0
else else
SLEEP_TIME=1
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_VERBOSE=1 _VERBOSE=1
fi fi
if [ "$SLEEP_TIME" == "" ]; then
SLEEP_TIME=.1
fi
SCRIPT_PID=$$ SCRIPT_PID=$$
LOCAL_USER=$(whoami) LOCAL_USER=$(whoami)
@ -80,7 +85,7 @@ fi
# Default alert attachment filename # Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$$.last.log"
# Set error exit code if a piped command fails # Set error exit code if a piped command fails
set -o pipefail set -o pipefail
@ -123,14 +128,20 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
ERROR_ALERT=1 ERROR_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value"
ERROR_ALERT=1 ERROR_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "WARN" ]; then elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value"
WARN_ALERT=1 WARN_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID"
return return
elif [ "$level" == "NOTICE" ]; then elif [ "$level" == "NOTICE" ]; then
_Logger "$prefix$value" _Logger "$prefix$value"
@ -176,33 +187,52 @@ function QuickLogger {
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds { function KillChilds {
local pid="${1}" local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" local self="${2:-false}" # Should parent be killed too ?
if children="$(pgrep -P "$pid")"; then # Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
for child in $children; do if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG Logger "Bogus pid given [$pid]." "CRITICAL"
KillChilds "$child" true return 1
done fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if kill -0 "$pid" > /dev/null 2>&1; then
if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
Logger "Sending SIGTERM to process [$pid]." "DEBUG" if children="$(pgrep -P "$pid")"; then
kill -s SIGTERM "$pid" if [[ "$pid" == *"$children"* ]]; then
if [ $? != 0 ]; then Logger "Bogus pgrep implementation." "CRITICAL"
sleep 15 children="${children/$pid/}"
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" fi
kill -9 "$pid" for child in $children; do
if [ $? != 0 ]; then Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" KillChilds "$child" true
return 1 done
fi fi
fi fi
return 0
else # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
return 0 if [ "$self" == true ]; then
fi # We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
} }
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
@ -297,7 +327,7 @@ function SendAlert {
fi fi
# Windows specific # Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then
Logger "Bogus smtp encryption, assuming none." "WARN" Logger "Bogus smtp encryption, assuming none." "WARN"
@ -310,14 +340,14 @@ function SendAlert {
if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\"" auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
fi fi
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0 return 0
fi fi
fi fi
# Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/) # Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/)
if type sendemail > /dev/null 2>&1 ; then if type sendemail > /dev/null 2>&1 ; then
@ -351,7 +381,7 @@ function SendAlert {
# Delete tmp log file # Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then if [ -f "$ALERT_LOG_FILE" ]; then
rm "$ALERT_LOG_FILE" rm -f "$ALERT_LOG_FILE"
fi fi
} }
@ -386,7 +416,7 @@ function SendEmail {
local auth_string= local auth_string=
if [ ! -f "$attachment" ]; then if [ ! -f "$attachment" ]; then
attachment_command="-a $ALERT_LOG_FILE" attachment_command="-a $attachment"
mail_no_attachment=1 mail_no_attachment=1
else else
mail_no_attachment=0 mail_no_attachment=0
@ -437,7 +467,7 @@ function SendEmail {
fi fi
# Windows specific # Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$sender_email" == "" ]; then if [ "$sender_email" == "" ]; then
Logger "Missing sender email." "ERROR" Logger "Missing sender email." "ERROR"
return 1 return 1
@ -461,14 +491,14 @@ function SendEmail {
if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then
auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\"" auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\""
fi fi
$(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string $(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0 return 0
fi fi
fi fi
# pfSense specific # pfSense specific
if [ -f /usr/local/bin/mail.php ]; then if [ -f /usr/local/bin/mail.php ]; then
@ -551,7 +581,7 @@ function Spinner {
# obsolete, use StripQuotes # obsolete, use StripQuotes
function SedStripQuotes { function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
} }
# Usage: var=$(StripSingleQuotes "$var") # Usage: var=$(StripSingleQuotes "$var")
@ -577,7 +607,7 @@ function StripQuotes {
function EscapeSpaces { function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }" echo "${string// /\\ }"
} }
function IsNumeric { function IsNumeric {
@ -767,10 +797,10 @@ function WaitForTaskCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
return 0
else else
return 1 Logger "Could not stop task" "ERROR"
fi fi
return 1
fi fi
fi fi
sleep $SLEEP_TIME sleep $SLEEP_TIME
@ -817,10 +847,16 @@ function WaitForCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
return 0
else else
return 1 Logger "Could not stop task" "ERROR"
fi fi
return 1
#if [ $? == 0 ]; then
# Logger "Task stopped successfully" "NOTICE"
# return 0
#else
# return 1
#fi
fi fi
fi fi
sleep $SLEEP_TIME sleep $SLEEP_TIME
@ -970,37 +1006,37 @@ function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing # Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
local number_of_arguments="${1}" # Number of arguments the tested function should have local number_of_arguments="${1}" # Number of arguments the tested function should have
local number_of_given_arguments="${2}" # Number of arguments that have been passed local number_of_given_arguments="${2}" # Number of arguments that have been passed
local function_name="${3}" # Function name that called __CheckArguments local function_name="${3}" # Function name that called __CheckArguments
if [ "$_PARANOIA_DEBUG" == "yes" ]; then if [ "$_PARANOIA_DEBUG" == "yes" ]; then
Logger "Entering function [$function_name]." "DEBUG" Logger "Entering function [$function_name]." "DEBUG"
fi fi
# All arguments of the function to check are passed as array in ${4} (the function call waits for $@) # All arguments of the function to check are passed as array in ${4} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments # If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${4} and count # In order to avoid this, we need to iterate over ${4} and count
local iterate=4 local iterate=4
local fetch_arguments=1 local fetch_arguments=1
local arg_list="" local arg_list=""
while [ $fetch_arguments -eq 1 ]; do while [ $fetch_arguments -eq 1 ]; do
cmd='argument=${'$iterate'}' cmd='argument=${'$iterate'}'
eval $cmd eval $cmd
if [ "$argument" = "" ]; then if [ "$argument" = "" ]; then
fetch_arguments=0 fetch_arguments=0
else else
arg_list="$arg_list [Argument $(($iterate-3)): $argument]" arg_list="$arg_list [Argument $(($iterate-3)): $argument]"
iterate=$(($iterate+1)) iterate=$(($iterate+1))
fi fi
done done
local counted_arguments=$((iterate-4)) local counted_arguments=$((iterate-4))
if [ $counted_arguments -ne $number_of_arguments ]; then if [ $counted_arguments -ne $number_of_arguments ]; then
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR" Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR"
Logger "Arguments passed: $arg_list" "ERROR" Logger "Arguments passed: $arg_list" "ERROR"
fi fi
fi fi
} }
@ -1011,7 +1047,7 @@ function RsyncPatternsAdd {
local pattern="${2}" local pattern="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local rest= local rest
# Disable globbing so wildcards from exclusions do not get expanded # Disable globbing so wildcards from exclusions do not get expanded
set -f set -f
@ -1037,221 +1073,228 @@ function RsyncPatternsAdd {
} }
function RsyncPatternsFromAdd { function RsyncPatternsFromAdd {
local pattern_type="${1}" local pattern_type="${1}"
local pattern_from="${2}" local pattern_from="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pattern_from= ## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
fi
## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ -e "$pattern_from" ]; then
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" fi
fi
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
fi
} }
function RsyncPatterns { function RsyncPatterns {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" fi
fi if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then fi
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then
fi RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then fi
RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" fi
fi # Use default include first for quicksync runs
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" elif [ "$RSYNC_PATTERN_FIRST" == "include" ] || [ "$_QUICK_SYNC" == "2" ]; then
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN"
fi fi
else if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi fi
if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
fi
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
fi
} }
function PreInit { function PreInit {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## SSH compression ## SSH compression
if [ "$SSH_COMPRESSION" != "no" ]; then if [ "$SSH_COMPRESSION" != "no" ]; then
SSH_COMP=-C SSH_COMP=-C
else else
SSH_COMP= SSH_COMP=
fi fi
## Ignore SSH known host verification ## Ignore SSH known host verification
if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then
SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
fi fi
## Support for older config files without RSYNC_EXECUTABLE option ## Support for older config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]; then if [ "$RSYNC_EXECUTABLE" == "" ]; then
RSYNC_EXECUTABLE=rsync RSYNC_EXECUTABLE=rsync
fi fi
## Sudo execution option ## Sudo execution option
if [ "$SUDO_EXEC" == "yes" ]; then if [ "$SUDO_EXEC" == "yes" ]; then
if [ "$RSYNC_REMOTE_PATH" != "" ]; then if [ "$RSYNC_REMOTE_PATH" != "" ]; then
RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"
else else
RSYNC_PATH="sudo $RSYNC_EXECUTABLE" RSYNC_PATH="sudo $RSYNC_EXECUTABLE"
fi fi
COMMAND_SUDO="sudo" COMMAND_SUDO="sudo"
else else
if [ "$RSYNC_REMOTE_PATH" != "" ]; then if [ "$RSYNC_REMOTE_PATH" != "" ]; then
RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"
else else
RSYNC_PATH="$RSYNC_EXECUTABLE" RSYNC_PATH="$RSYNC_EXECUTABLE"
fi fi
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set rsync default arguments
RSYNC_ARGS="-rltD" RSYNC_ARGS="-rltD"
RSYNC_ATTR_ARGS="-pgo" RSYNC_ATTR_ARGS="-pgo"
if [ "$_DRYRUN" -eq 1 ]; then if [ "$_DRYRUN" -eq 1 ]; then
RSYNC_DRY_ARG="-n" RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN" DRY_WARNING="/!\ DRY RUN"
else else
RSYNC_DRY_ARG="" RSYNC_DRY_ARG=""
fi fi
if [ "$PRESERVE_ACL" == "yes" ]; then if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi fi
if [ "$PRESERVE_XATTR" == "yes" ]; then if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z" RSYNC_ARGS=$RSYNC_ARGS" -z"
fi fi
if [ "$COPY_SYMLINKS" == "yes" ]; then if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L" RSYNC_ARGS=$RSYNC_ARGS" -L"
fi fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K" RSYNC_ARGS=$RSYNC_ARGS" -K"
fi fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H" RSYNC_ARGS=$RSYNC_ARGS" -H"
fi fi
if [ "$CHECKSUM" == "yes" ]; then if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi fi
if [ "$PARTIAL" == "yes" ]; then if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi fi
if [ "$DELTA_COPIES" != "no" ]; then if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file" RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi fi
## Set compression executable and extension ## Set compression executable and extension
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
if type xz > /dev/null 2>&1 if type xz > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.xz COMPRESSION_EXTENSION=.xz
elif type lzma > /dev/null 2>&1 elif type lzma > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.lzma COMPRESSION_EXTENSION=.lzma
elif type pigz > /dev/null 2>&1 elif type pigz > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.gz COMPRESSION_EXTENSION=.gz
# obackup specific # obackup specific
COMPRESSION_OPTIONS=--rsyncable COMPRESSION_OPTIONS=--rsyncable
elif type gzip > /dev/null 2>&1 elif type gzip > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.gz COMPRESSION_EXTENSION=.gz
# obackup specific # obackup specific
COMPRESSION_OPTIONS=--rsyncable COMPRESSION_OPTIONS=--rsyncable
else else
COMPRESSION_PROGRAM= COMPRESSION_PROGRAM=
COMPRESSION_EXTENSION= COMPRESSION_EXTENSION=
fi fi
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
} }
function PostInit { function PostInit {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
# Define remote commands # Define remote commands
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
} }
function InitLocalOSSettings { function InitLocalOSSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## If running under Msys, some commands do not run the same way ## If running under Msys, some commands do not run the same way
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ]; then if [ "$LOCAL_OS" == "msys" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
# PROCESS_TEST_CMD assumes there is a variable $pid # PROCESS_TEST_CMD assumes there is a variable $pid
# Tested on MSYS and cygwin # Tested on MSYS and cygwin
PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid'
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
else else
FIND_CMD=find FIND_CMD=find
# PROCESS_TEST_CMD assumes there is a variable $pid # PROCESS_TEST_CMD assumes there is a variable $pid
PROCESS_TEST_CMD='ps -p$pid' PROCESS_TEST_CMD='ps -p$pid'
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
STAT_CMD="stat --format %y" STAT_CMD="stat --format %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
} }
function InitRemoteOSSettings { function InitRemoteOSSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
REMOTE_FIND_CMD=find REMOTE_FIND_CMD=find
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
REMOTE_STAT_CMD="stat -f \"%Sm\"" REMOTE_STAT_CMD="stat -f \"%Sm\""
REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\""
else else
REMOTE_STAT_CMD="stat --format %y" REMOTE_STAT_CMD="stat --format %y"
REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\""
fi fi
} }
@ -1276,7 +1319,15 @@ function TrapStop {
} }
function TrapQuit { function TrapQuit {
local exitcode= local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=1
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=1
fi
if [ $ERROR_ALERT -ne 0 ]; then if [ $ERROR_ALERT -ne 0 ]; then
UnlockReplicas UnlockReplicas
@ -1308,9 +1359,7 @@ function TrapQuit {
exitcode=240 # Special exit code for daemon mode not stopping on warnings exitcode=240 # Special exit code for daemon mode not stopping on warnings
else else
UnlockReplicas UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then RunAfterHook
RunAfterHook
fi
CleanUp CleanUp
Logger "$PROGRAM finished." "NOTICE" Logger "$PROGRAM finished." "NOTICE"
exitcode=0 exitcode=0
@ -1631,7 +1680,7 @@ function _CheckLocksRemote {
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? == 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID) lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)
else else
@ -1826,9 +1875,8 @@ function _get_file_ctime_mtime_local {
local file_list="${3}" # Contains list of files to get time attrs local file_list="${3}" # Contains list of files to get time attrs
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
#cat "$file_list" | xargs -I {} stat -c '%n;%Z;%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list" while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
} }
function _get_file_ctime_mtime_remote { function _get_file_ctime_mtime_remote {
@ -1837,7 +1885,8 @@ function _get_file_ctime_mtime_remote {
local file_list="${3}" local file_list="${3}"
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd= local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"' cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG" Logger "CMD: $cmd" "DEBUG"
eval $cmd eval $cmd
@ -2041,18 +2090,19 @@ function _delete_local {
do do
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ "$SOFT_DELETE" != "no" ]; then if [ "$SOFT_DELETE" != "no" ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $replica_dir$files" "NOTICE" Logger "Soft deleting $replica_dir$files" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ -e "$replica_dir$deletion_dir/$files" ]; then if [ -e "$replica_dir$deletion_dir/$files" ]; then
rm -rf "${replica_dir:?}$deletion_dir/$files" rm -rf "${replica_dir:?}$deletion_dir/$files"
fi fi
@ -2108,7 +2158,7 @@ function _delete_remote {
# Additionnaly, we need to copy the deletetion list to the remote state folder # Additionnaly, we need to copy the deletetion list to the remote state folder
esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")" esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$2\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE" eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -2119,12 +2169,12 @@ function _delete_remote {
exit 1 exit 1
fi fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$TARGET_STATE_DIR/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 & $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
## The following lines are executed remotely ## The following lines are executed remotely
function _logger { function _logger {
local value="${1}" # What to log local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE" echo -e "$value" >&2 # Log to STDERR
if [ $_SILENT -eq 0 ]; then if [ $_SILENT -eq 0 ]; then
echo -e "$value" echo -e "$value"
@ -2168,32 +2218,36 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
IFS=$'\r\n' IFS=$'\r\n'
for files in $(cat "$FILE_LIST") for files in $(cat "$FILE_LIST")
do do
Logger "Processing file [$file]." "DEBUG"
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ "$SOFT_DELETE" != "no" ]; then if [ "$SOFT_DELETE" != "no" ]; then
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE" Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
fi fi
Logger "Full path for deletion is [$REPLICA_DIR$DELETE_DIR/$files]." "DEBUG"
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files" $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
fi fi
if [ -e "$$REPLICA_DIR$files" ]; then if [ -e "$REPLICA_DIR$files" ]; then
# In order to keep full path on soft deletion, create parent directories before move # In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")" parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then if [ "$parentdir" != "." ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir" $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
else else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"1 $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
fi fi
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR" Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
@ -2211,7 +2265,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files" $COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot delete $REPLICA_DIR$files" "ERROR" Logger "Cannot delete $REPLICA_DIR$files" "ERROR"
echo "$files" >> "$TARGET_STATE_DIR/$FAILED_DELETE_LIST" echo "$files" >> "$FAILED_DELETE_LIST"
fi fi
fi fi
fi fi
@ -2227,7 +2281,7 @@ ENDSSH
## Copy back the deleted failed file list ## Copy back the deleted failed file list
esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\"" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE" eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -2307,8 +2361,6 @@ function Sync {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
Logger "Starting synchronization task." "NOTICE" Logger "Starting synchronization task." "NOTICE"
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ -f "${INITIATOR[7]}" ] && [ "$RESUME_SYNC" != "no" ]; then if [ -f "${INITIATOR[7]}" ] && [ "$RESUME_SYNC" != "no" ]; then
resume_sync=$(cat "${INITIATOR[7]}") resume_sync=$(cat "${INITIATOR[7]}")
@ -2488,13 +2540,13 @@ function _SoftDeleteLocal {
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
else else
Dummy & Dummy &
fi fi
@ -2523,14 +2575,14 @@ function _SoftDeleteRemote {
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
if [ $_DRYRUN -eq 1 ]; then if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Does not remove anything." "NOTICE" Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
else else
Logger "Removing files older than $change_time days on target replica." "NOTICE" Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -2538,7 +2590,7 @@ function _SoftDeleteRemote {
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
@ -2548,10 +2600,10 @@ function _SoftDeleteRemote {
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR" Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
else else
Logger "Cleanup complete on target replica." "NOTICE" Logger "Cleanup complete on $replica_type replica." "NOTICE"
fi fi
} }
@ -2710,14 +2762,12 @@ function Init {
fi fi
## Add Rsync include / exclude patterns ## Add Rsync include / exclude patterns
if [ $_QUICK_SYNC -lt 2 ]; then RsyncPatterns
RsyncPatterns
fi
## Conflict options ## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[1]}${INITIATOR[4]}\"" INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[4]}\""
TARGET_BACKUP="--backup --backup-dir=\"${TARGET[1]}${TARGET[4]}\"" TARGET_BACKUP="--backup --backup-dir=\"${TARGET[4]}\""
if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then
INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
@ -2893,7 +2943,6 @@ do
;; ;;
--initiator=*) --initiator=*)
_QUICK_SYNC=$(($_QUICK_SYNC + 1)) _QUICK_SYNC=$(($_QUICK_SYNC + 1))
no_maxtime=1
INITIATOR_SYNC_DIR=${i##*=} INITIATOR_SYNC_DIR=${i##*=}
opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\"" opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\""
;; ;;
@ -2901,7 +2950,6 @@ do
_QUICK_SYNC=$(($_QUICK_SYNC + 1)) _QUICK_SYNC=$(($_QUICK_SYNC + 1))
TARGET_SYNC_DIR=${i##*=} TARGET_SYNC_DIR=${i##*=}
opts=$opts" --target=\"$TARGET_SYNC_DIR\"" opts=$opts" --target=\"$TARGET_SYNC_DIR\""
no_maxtime=1
;; ;;
--rsakey=*) --rsakey=*)
SSH_RSA_PRIVATE_KEY=${i##*=} SSH_RSA_PRIVATE_KEY=${i##*=}
@ -2964,6 +3012,10 @@ opts="${opts# *}"
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi
MIN_WAIT=30 MIN_WAIT=30
REMOTE_OPERATION=no REMOTE_OPERATION=no
else else
@ -3009,7 +3061,7 @@ opts="${opts# *}"
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
if [ $no_maxtime -eq 1 ]; then if [ $no_maxtime -eq 1 ] ; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
@ -3020,5 +3072,4 @@ opts="${opts# *}"
if [ $? == 0 ]; then if [ $? == 0 ]; then
SoftDelete SoftDelete
fi fi
RunAfterHook
fi fi

View File

@ -1,26 +1,38 @@
#!/usr/bin/env bash #!/usr/bin/env bash
## Merges ofunctions.sh and $PROGRAM ## MERGE 2016080601-b
## Merges ofunctions.sh and n_program.sh into program.sh
## Adds installer
PROGRAM=osync PROGRAM=osync
VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh) VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh)
VERSION=${VERSION#*=} VERSION=${VERSION#*=}
FUNC_PATH=/home/git/common
PARANOIA_DEBUG_LINE="#__WITH_PARANOIA_DEBUG" PARANOIA_DEBUG_LINE="__WITH_PARANOIA_DEBUG"
PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG" PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG"
PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG" PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG"
MINIMUM_FUNCTION_BEGIN="#### MINIMAL-FUNCTION-SET BEGIN ####"
MINIMUM_FUNCTION_END="#### MINIMAL-FUNCTION-SET END ####"
function Unexpand { function Unexpand {
unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh
} }
function Merge { function MergeAll {
sed "/source \"\.\/ofunctions.sh\"/r /home/git/common/ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh sed "/source \"\.\/ofunctions.sh\"/r ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh
chmod +x debug_$PROGRAM.sh
}
function MergeMinimum {
sed -n "/$MINIMUM_FUNCTION_BEGIN/,/$MINIMUM_FUNCTION_END/p" ofunctions.sh > tmp_minimal.sh
sed "/source \"\.\/ofunctions.sh\"/r tmp_minimal.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' | grep -v "$PARANOIA_DEBUG_LINE" > debug_$PROGRAM.sh
rm -f tmp_minimal.sh
chmod +x debug_$PROGRAM.sh chmod +x debug_$PROGRAM.sh
} }
function CleanDebug { function CleanDebug {
# sed explanation # sed explanation
@ -33,21 +45,27 @@ function CleanDebug {
#} #}
#p #p
sed -n '/'$PARANOIA_DEBUG_BEGIN'/{p; :a; N; /'$PARANOIA_DEBUG_END'/!ba; s/.*\n//}; p' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh sed '/'$PARANOIA_DEBUG_BEGIN'/,/'$PARANOIA_DEBUG_END'/d' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh
chmod +x ../$PROGRAM.sh chmod +x ../$PROGRAM.sh
} }
function CopyCommons { function CopyCommons {
sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_install.sh > ../tmp_install.sh sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../tmp_install.sh
sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh
sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_batch.sh > ../$PROGRAM-batch.sh if [ -f "common_batch.sh" ]; then
chmod +x ../install.sh sed "s/\[prgname\]/$PROGRAM/g" common_batch.sh > ../$PROGRAM-batch.sh
chmod +x ../$PROGRAM-batch.sh fi
chmod +x ../install.sh
chmod +x ../$PROGRAM-batch.sh
rm -f ../tmp_install.sh
} }
Unexpand Unexpand
Merge if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
MergeAll
else
MergeMinimum
fi
CleanDebug CleanDebug
rm -f tmp_$PROGRAM.sh
rm -f ../tmp_install.sh
CopyCommons CopyCommons
rm -f tmp_$PROGRAM.sh

View File

@ -3,8 +3,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1 PROGRAM_VERSION=1.1.6-rc1
PROGRAM_BUILD=2016072701 PROGRAM_BUILD=2017060801
IS_STABLE=yes IS_STABLE=yes
source "./ofunctions.sh" source "./ofunctions.sh"
@ -28,7 +28,15 @@ function TrapStop {
} }
function TrapQuit { function TrapQuit {
local exitcode= local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=1
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=1
fi
if [ $ERROR_ALERT -ne 0 ]; then if [ $ERROR_ALERT -ne 0 ]; then
UnlockReplicas UnlockReplicas
@ -60,9 +68,7 @@ function TrapQuit {
exitcode=240 # Special exit code for daemon mode not stopping on warnings exitcode=240 # Special exit code for daemon mode not stopping on warnings
else else
UnlockReplicas UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then RunAfterHook
RunAfterHook
fi
CleanUp CleanUp
Logger "$PROGRAM finished." "NOTICE" Logger "$PROGRAM finished." "NOTICE"
exitcode=0 exitcode=0
@ -383,7 +389,7 @@ function _CheckLocksRemote {
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? == 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID) lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)
else else
@ -514,9 +520,9 @@ function tree_list {
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "${TARGET[0]}" ]; then if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "${TARGET[0]}" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | sed -r 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE" ## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE"
@ -578,9 +584,8 @@ function _get_file_ctime_mtime_local {
local file_list="${3}" # Contains list of files to get time attrs local file_list="${3}" # Contains list of files to get time attrs
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
#cat "$file_list" | xargs -I {} stat -c '%n;%Z;%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list" while IFS='' read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
} }
function _get_file_ctime_mtime_remote { function _get_file_ctime_mtime_remote {
@ -589,8 +594,9 @@ function _get_file_ctime_mtime_remote {
local file_list="${3}" local file_list="${3}"
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd= local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
cmd='cat "'$file_list'" | '$SSH_CMD' "while IFS=\"\" read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG" Logger "CMD: $cmd" "DEBUG"
eval $cmd eval $cmd
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -793,18 +799,19 @@ function _delete_local {
do do
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ "$SOFT_DELETE" != "no" ]; then if [ "$SOFT_DELETE" != "no" ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $replica_dir$files" "NOTICE" Logger "Soft deleting $replica_dir$files" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ -e "$replica_dir$deletion_dir/$files" ]; then if [ -e "$replica_dir$deletion_dir/$files" ]; then
rm -rf "${replica_dir:?}$deletion_dir/$files" rm -rf "${replica_dir:?}$deletion_dir/$files"
fi fi
@ -860,7 +867,7 @@ function _delete_remote {
# Additionnaly, we need to copy the deletetion list to the remote state folder # Additionnaly, we need to copy the deletetion list to the remote state folder
esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")" esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$2\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE" eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -871,12 +878,12 @@ function _delete_remote {
exit 1 exit 1
fi fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$TARGET_STATE_DIR/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 & $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
## The following lines are executed remotely ## The following lines are executed remotely
function _logger { function _logger {
local value="${1}" # What to log local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE" echo -e "$value" >&2 # Log to STDERR
if [ $_SILENT -eq 0 ]; then if [ $_SILENT -eq 0 ]; then
echo -e "$value" echo -e "$value"
@ -920,32 +927,36 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
IFS=$'\r\n' IFS=$'\r\n'
for files in $(cat "$FILE_LIST") for files in $(cat "$FILE_LIST")
do do
Logger "Processing file [$file]." "DEBUG"
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ "$SOFT_DELETE" != "no" ]; then if [ "$SOFT_DELETE" != "no" ]; then
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE" Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
fi fi
Logger "Full path for deletion is [$REPLICA_DIR$DELETE_DIR/$files]." "DEBUG"
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files" $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
fi fi
if [ -e "$$REPLICA_DIR$files" ]; then if [ -e "$REPLICA_DIR$files" ]; then
# In order to keep full path on soft deletion, create parent directories before move # In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")" parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then if [ "$parentdir" != "." ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir" $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
else else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"1 $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
fi fi
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR" Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
@ -963,7 +974,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files" $COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot delete $REPLICA_DIR$files" "ERROR" Logger "Cannot delete $REPLICA_DIR$files" "ERROR"
echo "$files" >> "$TARGET_STATE_DIR/$FAILED_DELETE_LIST" echo "$files" >> "$FAILED_DELETE_LIST"
fi fi
fi fi
fi fi
@ -979,7 +990,7 @@ ENDSSH
## Copy back the deleted failed file list ## Copy back the deleted failed file list
esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\"" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE" eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -1059,8 +1070,6 @@ function Sync {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
Logger "Starting synchronization task." "NOTICE" Logger "Starting synchronization task." "NOTICE"
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ -f "${INITIATOR[7]}" ] && [ "$RESUME_SYNC" != "no" ]; then if [ -f "${INITIATOR[7]}" ] && [ "$RESUME_SYNC" != "no" ]; then
resume_sync=$(cat "${INITIATOR[7]}") resume_sync=$(cat "${INITIATOR[7]}")
@ -1240,13 +1249,13 @@ function _SoftDeleteLocal {
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
else else
Dummy & Dummy &
fi fi
@ -1275,14 +1284,14 @@ function _SoftDeleteRemote {
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
if [ $_DRYRUN -eq 1 ]; then if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Does not remove anything." "NOTICE" Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
else else
Logger "Removing files older than $change_time days on target replica." "NOTICE" Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -1290,7 +1299,7 @@ function _SoftDeleteRemote {
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
@ -1300,10 +1309,10 @@ function _SoftDeleteRemote {
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR" Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
else else
Logger "Cleanup complete on target replica." "NOTICE" Logger "Cleanup complete on $replica_type replica." "NOTICE"
fi fi
} }
@ -1462,14 +1471,12 @@ function Init {
fi fi
## Add Rsync include / exclude patterns ## Add Rsync include / exclude patterns
if [ $_QUICK_SYNC -lt 2 ]; then RsyncPatterns
RsyncPatterns
fi
## Conflict options ## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[1]}${INITIATOR[4]}\"" INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[4]}\""
TARGET_BACKUP="--backup --backup-dir=\"${TARGET[1]}${TARGET[4]}\"" TARGET_BACKUP="--backup --backup-dir=\"${TARGET[4]}\""
if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then
INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
@ -1645,7 +1652,6 @@ do
;; ;;
--initiator=*) --initiator=*)
_QUICK_SYNC=$(($_QUICK_SYNC + 1)) _QUICK_SYNC=$(($_QUICK_SYNC + 1))
no_maxtime=1
INITIATOR_SYNC_DIR=${i##*=} INITIATOR_SYNC_DIR=${i##*=}
opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\"" opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\""
;; ;;
@ -1653,7 +1659,6 @@ do
_QUICK_SYNC=$(($_QUICK_SYNC + 1)) _QUICK_SYNC=$(($_QUICK_SYNC + 1))
TARGET_SYNC_DIR=${i##*=} TARGET_SYNC_DIR=${i##*=}
opts=$opts" --target=\"$TARGET_SYNC_DIR\"" opts=$opts" --target=\"$TARGET_SYNC_DIR\""
no_maxtime=1
;; ;;
--rsakey=*) --rsakey=*)
SSH_RSA_PRIVATE_KEY=${i##*=} SSH_RSA_PRIVATE_KEY=${i##*=}
@ -1716,6 +1721,10 @@ opts="${opts# *}"
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi
MIN_WAIT=30 MIN_WAIT=30
REMOTE_OPERATION=no REMOTE_OPERATION=no
else else
@ -1761,7 +1770,7 @@ opts="${opts# *}"
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
if [ $no_maxtime -eq 1 ]; then if [ $no_maxtime -eq 1 ] ; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
@ -1772,5 +1781,4 @@ opts="${opts# *}"
if [ $? == 0 ]; then if [ $? == 0 ]; then
SoftDelete SoftDelete
fi fi
RunAfterHook
fi fi

View File

@ -1,4 +1,4 @@
## FUNC_BUILD=2016071902 ## FUNC_BUILD=2016071902-j
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -15,6 +15,9 @@ KEEP_LOGGING=1801
## Correct output of sort command (language agnostic sorting) ## Correct output of sort command (language agnostic sorting)
export LC_ALL=C export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body # Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -38,14 +41,16 @@ fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.1
_VERBOSE=0 _VERBOSE=0
else else
SLEEP_TIME=1
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_VERBOSE=1 _VERBOSE=1
fi fi
if [ "$SLEEP_TIME" == "" ]; then
SLEEP_TIME=.1
fi
SCRIPT_PID=$$ SCRIPT_PID=$$
LOCAL_USER=$(whoami) LOCAL_USER=$(whoami)
@ -71,7 +76,7 @@ fi
# Default alert attachment filename # Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$$.last.log"
# Set error exit code if a piped command fails # Set error exit code if a piped command fails
set -o pipefail set -o pipefail
@ -114,14 +119,20 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
ERROR_ALERT=1 ERROR_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value"
ERROR_ALERT=1 ERROR_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "WARN" ]; then elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value"
WARN_ALERT=1 WARN_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID"
return return
elif [ "$level" == "NOTICE" ]; then elif [ "$level" == "NOTICE" ]; then
_Logger "$prefix$value" _Logger "$prefix$value"
@ -167,33 +178,52 @@ function QuickLogger {
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds { function KillChilds {
local pid="${1}" local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" local self="${2:-false}" # Should parent be killed too ?
if children="$(pgrep -P "$pid")"; then # Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
for child in $children; do if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG Logger "Bogus pid given [$pid]." "CRITICAL"
KillChilds "$child" true return 1
done fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if kill -0 "$pid" > /dev/null 2>&1; then
if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
Logger "Sending SIGTERM to process [$pid]." "DEBUG" if children="$(pgrep -P "$pid")"; then
kill -s SIGTERM "$pid" if [[ "$pid" == *"$children"* ]]; then
if [ $? != 0 ]; then Logger "Bogus pgrep implementation." "CRITICAL"
sleep 15 children="${children/$pid/}"
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" fi
kill -9 "$pid" for child in $children; do
if [ $? != 0 ]; then Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" KillChilds "$child" true
return 1 done
fi fi
fi fi
return 0
else # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
return 0 if [ "$self" == true ]; then
fi # We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
} }
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
@ -288,7 +318,7 @@ function SendAlert {
fi fi
# Windows specific # Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then
Logger "Bogus smtp encryption, assuming none." "WARN" Logger "Bogus smtp encryption, assuming none." "WARN"
@ -301,14 +331,14 @@ function SendAlert {
if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\"" auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
fi fi
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0 return 0
fi fi
fi fi
# Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/) # Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/)
if type sendemail > /dev/null 2>&1 ; then if type sendemail > /dev/null 2>&1 ; then
@ -342,7 +372,7 @@ function SendAlert {
# Delete tmp log file # Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then if [ -f "$ALERT_LOG_FILE" ]; then
rm "$ALERT_LOG_FILE" rm -f "$ALERT_LOG_FILE"
fi fi
} }
@ -377,7 +407,7 @@ function SendEmail {
local auth_string= local auth_string=
if [ ! -f "$attachment" ]; then if [ ! -f "$attachment" ]; then
attachment_command="-a $ALERT_LOG_FILE" attachment_command="-a $attachment"
mail_no_attachment=1 mail_no_attachment=1
else else
mail_no_attachment=0 mail_no_attachment=0
@ -428,7 +458,7 @@ function SendEmail {
fi fi
# Windows specific # Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$sender_email" == "" ]; then if [ "$sender_email" == "" ]; then
Logger "Missing sender email." "ERROR" Logger "Missing sender email." "ERROR"
return 1 return 1
@ -452,14 +482,14 @@ function SendEmail {
if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then
auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\"" auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\""
fi fi
$(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string $(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0 return 0
fi fi
fi fi
# pfSense specific # pfSense specific
if [ -f /usr/local/bin/mail.php ]; then if [ -f /usr/local/bin/mail.php ]; then
@ -542,7 +572,7 @@ function Spinner {
# obsolete, use StripQuotes # obsolete, use StripQuotes
function SedStripQuotes { function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
} }
# Usage: var=$(StripSingleQuotes "$var") # Usage: var=$(StripSingleQuotes "$var")
@ -568,7 +598,7 @@ function StripQuotes {
function EscapeSpaces { function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }" echo "${string// /\\ }"
} }
function IsNumeric { function IsNumeric {
@ -636,7 +666,7 @@ function GetLocalOS {
*"BSD"*) *"BSD"*)
LOCAL_OS="BSD" LOCAL_OS="BSD"
;; ;;
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN"*)
LOCAL_OS="msys" LOCAL_OS="msys"
;; ;;
*"Darwin"*) *"Darwin"*)
@ -696,7 +726,7 @@ function GetRemoteOS {
*"BSD"*) *"BSD"*)
REMOTE_OS="BSD" REMOTE_OS="BSD"
;; ;;
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN"*)
REMOTE_OS="msys" REMOTE_OS="msys"
;; ;;
*"Darwin"*) *"Darwin"*)
@ -758,10 +788,10 @@ function WaitForTaskCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
return 0
else else
return 1 Logger "Could not stop task" "ERROR"
fi fi
return 1
fi fi
fi fi
sleep $SLEEP_TIME sleep $SLEEP_TIME
@ -808,10 +838,16 @@ function WaitForCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
return 0
else else
return 1 Logger "Could not stop task" "ERROR"
fi fi
return 1
#if [ $? == 0 ]; then
# Logger "Task stopped successfully" "NOTICE"
# return 0
#else
# return 1
#fi
fi fi
fi fi
sleep $SLEEP_TIME sleep $SLEEP_TIME
@ -961,37 +997,37 @@ function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing # Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then if [ "$_DEBUG" == "yes" ]; then
local number_of_arguments="${1}" # Number of arguments the tested function should have local number_of_arguments="${1}" # Number of arguments the tested function should have
local number_of_given_arguments="${2}" # Number of arguments that have been passed local number_of_given_arguments="${2}" # Number of arguments that have been passed
local function_name="${3}" # Function name that called __CheckArguments local function_name="${3}" # Function name that called __CheckArguments
if [ "$_PARANOIA_DEBUG" == "yes" ]; then if [ "$_PARANOIA_DEBUG" == "yes" ]; then
Logger "Entering function [$function_name]." "DEBUG" Logger "Entering function [$function_name]." "DEBUG"
fi fi
# All arguments of the function to check are passed as array in ${4} (the function call waits for $@) # All arguments of the function to check are passed as array in ${4} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments # If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${4} and count # In order to avoid this, we need to iterate over ${4} and count
local iterate=4 local iterate=4
local fetch_arguments=1 local fetch_arguments=1
local arg_list="" local arg_list=""
while [ $fetch_arguments -eq 1 ]; do while [ $fetch_arguments -eq 1 ]; do
cmd='argument=${'$iterate'}' cmd='argument=${'$iterate'}'
eval $cmd eval $cmd
if [ "$argument" = "" ]; then if [ "$argument" = "" ]; then
fetch_arguments=0 fetch_arguments=0
else else
arg_list="$arg_list [Argument $(($iterate-3)): $argument]" arg_list="$arg_list [Argument $(($iterate-3)): $argument]"
iterate=$(($iterate+1)) iterate=$(($iterate+1))
fi fi
done done
local counted_arguments=$((iterate-4)) local counted_arguments=$((iterate-4))
if [ $counted_arguments -ne $number_of_arguments ]; then if [ $counted_arguments -ne $number_of_arguments ]; then
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR" Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR"
Logger "Arguments passed: $arg_list" "ERROR" Logger "Arguments passed: $arg_list" "ERROR"
fi fi
fi fi
} }
@ -1002,7 +1038,7 @@ function RsyncPatternsAdd {
local pattern="${2}" local pattern="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local rest= local rest
# Disable globbing so wildcards from exclusions do not get expanded # Disable globbing so wildcards from exclusions do not get expanded
set -f set -f
@ -1028,221 +1064,228 @@ function RsyncPatternsAdd {
} }
function RsyncPatternsFromAdd { function RsyncPatternsFromAdd {
local pattern_type="${1}" local pattern_type="${1}"
local pattern_from="${2}" local pattern_from="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pattern_from= ## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
fi
## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ -e "$pattern_from" ]; then
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" fi
fi
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
fi
} }
function RsyncPatterns { function RsyncPatterns {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" fi
fi if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then fi
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then
fi RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then fi
RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" fi
fi # Use default include first for quicksync runs
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" elif [ "$RSYNC_PATTERN_FIRST" == "include" ] || [ "$_QUICK_SYNC" == "2" ]; then
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN"
fi fi
else if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi fi
if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
fi
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
fi
} }
function PreInit { function PreInit {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## SSH compression ## SSH compression
if [ "$SSH_COMPRESSION" != "no" ]; then if [ "$SSH_COMPRESSION" != "no" ]; then
SSH_COMP=-C SSH_COMP=-C
else else
SSH_COMP= SSH_COMP=
fi fi
## Ignore SSH known host verification ## Ignore SSH known host verification
if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then
SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
fi fi
## Support for older config files without RSYNC_EXECUTABLE option ## Support for older config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]; then if [ "$RSYNC_EXECUTABLE" == "" ]; then
RSYNC_EXECUTABLE=rsync RSYNC_EXECUTABLE=rsync
fi fi
## Sudo execution option ## Sudo execution option
if [ "$SUDO_EXEC" == "yes" ]; then if [ "$SUDO_EXEC" == "yes" ]; then
if [ "$RSYNC_REMOTE_PATH" != "" ]; then if [ "$RSYNC_REMOTE_PATH" != "" ]; then
RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"
else else
RSYNC_PATH="sudo $RSYNC_EXECUTABLE" RSYNC_PATH="sudo $RSYNC_EXECUTABLE"
fi fi
COMMAND_SUDO="sudo" COMMAND_SUDO="sudo"
else else
if [ "$RSYNC_REMOTE_PATH" != "" ]; then if [ "$RSYNC_REMOTE_PATH" != "" ]; then
RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"
else else
RSYNC_PATH="$RSYNC_EXECUTABLE" RSYNC_PATH="$RSYNC_EXECUTABLE"
fi fi
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set rsync default arguments
RSYNC_ARGS="-rltD" RSYNC_ARGS="-rltD"
RSYNC_ATTR_ARGS="-pgo" RSYNC_ATTR_ARGS="-pgo"
if [ "$_DRYRUN" -eq 1 ]; then if [ "$_DRYRUN" -eq 1 ]; then
RSYNC_DRY_ARG="-n" RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN" DRY_WARNING="/!\ DRY RUN"
else else
RSYNC_DRY_ARG="" RSYNC_DRY_ARG=""
fi fi
if [ "$PRESERVE_ACL" == "yes" ]; then if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi fi
if [ "$PRESERVE_XATTR" == "yes" ]; then if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z" RSYNC_ARGS=$RSYNC_ARGS" -z"
fi fi
if [ "$COPY_SYMLINKS" == "yes" ]; then if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L" RSYNC_ARGS=$RSYNC_ARGS" -L"
fi fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K" RSYNC_ARGS=$RSYNC_ARGS" -K"
fi fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H" RSYNC_ARGS=$RSYNC_ARGS" -H"
fi fi
if [ "$CHECKSUM" == "yes" ]; then if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi fi
if [ "$PARTIAL" == "yes" ]; then if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi fi
if [ "$DELTA_COPIES" != "no" ]; then if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file" RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi fi
## Set compression executable and extension ## Set compression executable and extension
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
if type xz > /dev/null 2>&1 if type xz > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.xz COMPRESSION_EXTENSION=.xz
elif type lzma > /dev/null 2>&1 elif type lzma > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.lzma COMPRESSION_EXTENSION=.lzma
elif type pigz > /dev/null 2>&1 elif type pigz > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.gz COMPRESSION_EXTENSION=.gz
# obackup specific # obackup specific
COMPRESSION_OPTIONS=--rsyncable COMPRESSION_OPTIONS=--rsyncable
elif type gzip > /dev/null 2>&1 elif type gzip > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.gz COMPRESSION_EXTENSION=.gz
# obackup specific # obackup specific
COMPRESSION_OPTIONS=--rsyncable COMPRESSION_OPTIONS=--rsyncable
else else
COMPRESSION_PROGRAM= COMPRESSION_PROGRAM=
COMPRESSION_EXTENSION= COMPRESSION_EXTENSION=
fi fi
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
} }
function PostInit { function PostInit {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
# Define remote commands # Define remote commands
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
} }
function InitLocalOSSettings { function InitLocalOSSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## If running under Msys, some commands do not run the same way ## If running under Msys, some commands do not run the same way
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ]; then if [ "$LOCAL_OS" == "msys" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
# PROCESS_TEST_CMD assumes there is a variable $pid # PROCESS_TEST_CMD assumes there is a variable $pid
# Tested on MSYS and cygwin # Tested on MSYS and cygwin
PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid'
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
else else
FIND_CMD=find FIND_CMD=find
# PROCESS_TEST_CMD assumes there is a variable $pid # PROCESS_TEST_CMD assumes there is a variable $pid
PROCESS_TEST_CMD='ps -p$pid' PROCESS_TEST_CMD='ps -p$pid'
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
STAT_CMD="stat --format %y" STAT_CMD="stat --format %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
} }
function InitRemoteOSSettings { function InitRemoteOSSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
REMOTE_FIND_CMD=find REMOTE_FIND_CMD=find
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
REMOTE_STAT_CMD="stat -f \"%Sm\"" REMOTE_STAT_CMD="stat -f \"%Sm\""
REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\""
else else
REMOTE_STAT_CMD="stat --format %y" REMOTE_STAT_CMD="stat --format %y"
REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\""
fi fi
} }

185
dev/tests/conf/local.conf Normal file
View File

@ -0,0 +1,185 @@
#!/usr/bin/env bash
###### osync - Rsync based two way sync engine with fault tolerance
###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
###### osync v1.1x / v1.2x config file rev 2016102101
## ---------- GENERAL OPTIONS
## Sync job identification
INSTANCE_ID="local"
## Directories to synchronize.
## Initiator is the system osync runs on. The initiator directory must be a local path.
INITIATOR_SYNC_DIR="${HOME}/osync-tests/initiator"
## Target is the system osync synchronizes to (can be the same system as the initiator in case of local sync tasks). The target directory can be a local or remote path.
TARGET_SYNC_DIR="${HOME}/osync-tests/target"
#TARGET_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir2"
## If the target system is remote, you can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa"
## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
SSH_PASSWORD_FILE=""
## Create sync directories if they do not exist
CREATE_DIRS=no
## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist)
LOGFILE=""
## Generate an alert if initiator or target replicas have less free space than given value in KB. Set this to zero to skip disk space tests.
MINIMUM_SPACE=10240
## Bandwidth limit Kbytes / second. Leave 0 to disable limitation
BANDWIDTH=0
## If enabled, synchronization on remote system will be processed as superuser. See documentation for /etc/sudoers file configuration.
SUDO_EXEC=no
## Paranoia option. Don't change this unless you read the documentation.
RSYNC_EXECUTABLE=rsync
## Remote rsync executable path. Leave this empty in most cases
RSYNC_REMOTE_PATH=""
## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
RSYNC_PATTERN_FIRST=include
## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_INCLUDE_PATTERN=""
RSYNC_EXCLUDE_PATTERN="*.php"
#RSYNC_EXCLUDE_PATTERN="tmp;archives"
## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## This file has to be in the same directory as the config file
## Paths are relative to sync dirs. One element per line.
RSYNC_INCLUDE_FROM=""
RSYNC_EXCLUDE_FROM=""
#RSYNC_EXCLUDE_FROM="exclude.list"
## List elements separator char. You may set an alternative separator char for your directories lists above.
PATH_SEPARATOR_CHAR=";"
## ---------- REMOTE SYNC OPTIONS
## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes
## Ignore ssh known hosts. DANGER WILL ROBINSON DANGER ! This can lead to security issues. Only enable this if you know what you're doing.
SSH_IGNORE_KNOWN_HOSTS=no
## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync.
REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync.
## If you use this function, you should set more than one 3rd party host, and be sure you can ping them.
## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute.
REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
## ---------- MISC OPTIONS
## Preserve basic linux permissions
PRESERVE_PERMISSIONS=yes
PRESERVE_OWNER=yes
PRESERVE_GROUP=yes
## On MACOS X, does not work and will be ignored
PRESERVE_EXECUTABILITY=yes
## Preserve ACLS. Make sure source and target FS can manage same ACLs or you'll get loads of errors.
PRESERVE_ACL=yes
## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors.
PRESERVE_XATTR=yes
## Transforms symlinks into referent files/dirs
COPY_SYMLINKS=no
## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
KEEP_DIRLINKS=no
## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
PRESERVE_HARDLINKS=no
## Do a full checksum on all files that have identical sizes, they are checksummed to see if they actually are identical. This can take a long time.
CHECKSUM=no
## Let RSYNC compress file transfers. Do not use this if both initator and target replicas are on local system. Also, do not use this if you already enabled SSH compression.
RSYNC_COMPRESS=yes
## Maximum execution time (in seconds) for sync process. Set these values zero will disable max execution times.
## Soft exec time only generates a warning. Hard exec time will generate a warning and stop sync process.
SOFT_MAX_EXEC_TIME=7200
HARD_MAX_EXEC_TIME=10600
## Log a message every KEEP_LOGGING seconds just to know the task is still alive
KEEP_LOGGING=1801
## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish.
MIN_WAIT=60
## Maximum time (in seconds) waiting in file monitor / daemon mode. After this time, sync is run.
## Use 0 to wait indefinitely.
MAX_WAIT=7200
## ---------- BACKUP AND DELETION OPTIONS
## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups
CONFLICT_BACKUP=yes
## Keep multiple backup versions of the same file. Warning, This can be very space consuming.
CONFLICT_BACKUP_MULTIPLE=no
## Osync will clean backup files after a given number of days. Setting this to 0 will disable cleaning and keep backups forever. Warning: This can be very space consuming.
CONFLICT_BACKUP_DAYS=30
## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica.
CONFLICT_PREVALANCE=initiator
## On deletion propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted
SOFT_DELETE=yes
## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming.
SOFT_DELETE_DAYS=30
## Optional deletion skip on replicas. Valid values are "initiator", "target", or "initiator,target"
SKIP_DELETION=
## ---------- RESUME OPTIONS
## Try to resume an aborted sync task
RESUME_SYNC=yes
## Number maximum resume tries before initiating a fresh sync.
RESUME_TRY=2
## When a pidlock exists on slave replica that does not correspond to the initiator's instance-id, force pidlock removal. Be careful with this option if you have multiple initiators.
FORCE_STRANGER_LOCK_RESUME=no
## Keep partial uploads that can be resumed on next run, experimental feature
PARTIAL=no
## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
DELTA_COPIES=yes
## ---------- ALERT OPTIONS
## List of alert mails separated by spaces
DESTINATION_MAILS=""
## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/)
SENDER_MAIL="alert@your.system.tld"
SMTP_SERVER=smtp.your.isp.tld
SMTP_PORT=25
# encryption can be tls, ssl or none
SMTP_ENCRYPTION=none
SMTP_USER=
SMTP_PASSWORD=
## ---------- EXECUTION HOOKS
## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_OPERATION is set).
LOCAL_RUN_BEFORE_CMD=""
LOCAL_RUN_AFTER_CMD=""
REMOTE_RUN_BEFORE_CMD=""
REMOTE_RUN_AFTER_CMD=""
## Max execution time of commands before they get force killed. Leave 0 if you don't wan't this to happen. Time is specified in seconds.
MAX_EXEC_TIME_PER_CMD_BEFORE=0
MAX_EXEC_TIME_PER_CMD_AFTER=0
## Stops osync execution if one of the above commands fail
STOP_ON_CMD_ERROR=yes
## Run local and remote after sync commands even on failure
RUN_AFTER_CMD_ON_ERROR=no

69
tests/conf/local.conf → dev/tests/conf/old.conf Executable file → Normal file
View File

@ -1,20 +1,17 @@
#!/usr/bin/env bash #!/bin/bash
###### Osync - Rsync based two way sync engine with fault tolerance ###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
###### Config file rev 2611201401 #### Config file rev 0211201302
## ---------- GENERAL OPTIONS ## ---------- GENERAL OPTIONS
## Sync job identification ## Sync job identification
SYNC_ID="sync_test" SYNC_ID="old"
## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one. ## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either on the same system, or on a remote one.
MASTER_SYNC_DIR="master" MASTER_SYNC_DIR="${HOME}/osync-tests/initiator"
SLAVE_SYNC_DIR="slave" SLAVE_SYNC_DIR="${HOME}/osync-tests/target"
#SLAVE_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir2"
## If slave replica is a remote directory, you must specify a RSA key (please use full path). Please see documentation for further information.
#SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa"
## Create sync directories if they do not exist ## Create sync directories if they do not exist
CREATE_DIRS=no CREATE_DIRS=no
@ -22,19 +19,10 @@ CREATE_DIRS=no
## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist) ## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist)
LOGFILE="" LOGFILE=""
## List of directories to exclude from sync on both sides (rsync patterns, wildcards work). ## List of directories to exclude from sync on both sides (rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon. ## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_EXCLUDE_PATTERN="" RSYNC_EXCLUDE_PATTERN="tmp;archives"
#RSYNC_EXCLUDE_PATTERN="tmp;archives" ## List elements separator char. You may set an alternative seperator char for your directories lists above.
## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## This file has to be in the same directory as the config file
## Paths are relative to sync dirs. One element per line.
RSYNC_EXCLUDE_FROM=""
#RSYNC_EXCLUDE_FROM="exclude.list"
## List elements separator char. You may set an alternative separator char for your directories lists above.
PATH_SEPARATOR_CHAR=";" PATH_SEPARATOR_CHAR=";"
## Generate an alert if master or slave replicas have less free space than given value in KB. ## Generate an alert if master or slave replicas have less free space than given value in KB.
@ -43,13 +31,20 @@ MINIMUM_SPACE=10240
## Bandwidth limit Kbytes / second. Leave 0 to disable limitation ## Bandwidth limit Kbytes / second. Leave 0 to disable limitation
BANDWIDTH=0 BANDWIDTH=0
## If enabled, synchronization on remote system will be processed as superuser. See documentation for /etc/sudoers file configuration. ## If enabled, synchronization will be processed as superuser. See documentation for /etc/sudoers file configuration.
SUDO_EXEC=no SUDO_EXEC=no
## Paranoia option. Don't change this unless you read the documentation. ## Paranoia option. Don't change this unless you read the documentation.
RSYNC_EXECUTABLE=rsync RSYNC_EXECUTABLE=rsync
## ---------- REMOTE SYNC OPTIONS ## ---------- REMOTE SYNC OPTIONS
## The following options allow Osync to sync a slave replica on a remote system via an SSH tunnel.
## Needs public RSA key need to be put into ~/.ssh/authorized_keys in remote users home directory. See documentation for remote sync.
REMOTE_SYNC=no
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
REMOTE_USER=syncuser
REMOTE_HOST=your-remote-host.tld
REMOTE_PORT=22
## ssh compression should be used unless your remote connection is good enough (LAN) ## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes SSH_COMPRESSION=yes
@ -59,10 +54,10 @@ REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync. ## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync.
## If you use this function, you should set more than one 3rd party host, and be sure you can ping them. ## If you use this function, you should set more than one 3rd party host, and be sure you can ping them.
## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute. ## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute.
REMOTE_3RD_PARTY_HOSTS="" REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.fr"
## Remote rsync executable path. Leave this empty in most cases ## Remote rsync executable path. Leave this empty in most cases
RSYNC_REMOTE_PATH="" REMOTE_RSYNC_PATH=""
## ---------- MISC OPTIONS ## ---------- MISC OPTIONS
@ -70,12 +65,6 @@ RSYNC_REMOTE_PATH=""
PRESERVE_ACL=no PRESERVE_ACL=no
## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors. ## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors.
PRESERVE_XATTR=no PRESERVE_XATTR=no
## Transforms symlinks into referent files/dirs
COPY_SYMLINKS=no
## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
KEEP_DIRLINKS=no
## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
PRESERVE_HARDLINKS=no
## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression. ## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression.
RSYNC_COMPRESS=yes RSYNC_COMPRESS=yes
@ -84,10 +73,7 @@ RSYNC_COMPRESS=yes
SOFT_MAX_EXEC_TIME=7200 SOFT_MAX_EXEC_TIME=7200
HARD_MAX_EXEC_TIME=10600 HARD_MAX_EXEC_TIME=10600
## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish. ## ---------- BACKUP AND TRASH OPTIONS
MIN_WAIT=60
## ---------- BACKUP AND DELETION OPTIONS
## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups ## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups
CONFLICT_BACKUP=yes CONFLICT_BACKUP=yes
@ -98,7 +84,7 @@ CONFLICT_BACKUP_DAYS=30
## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica. ## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica.
CONFLICT_PREVALANCE=master CONFLICT_PREVALANCE=master
## On deletion propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted ## On deletition propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted
SOFT_DELETE=yes SOFT_DELETE=yes
## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming. ## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming.
SOFT_DELETE_DAYS=30 SOFT_DELETE_DAYS=30
@ -107,22 +93,19 @@ SOFT_DELETE_DAYS=30
## Try to resume an aborted sync task ## Try to resume an aborted sync task
RESUME_SYNC=yes RESUME_SYNC=yes
## Number maximum resume tries before initiating a fresh sync. ## Number maximum resume tries before initating a fresh sync.
RESUME_TRY=2 RESUME_TRY=2
## When a pidlock exists on slave replica that does not correspond to master's sync-id, force pidlock removal. Be careful with this option if you have multiple masters. ## When a pidlock exists on slave replica that does not correspond to master's sync-id, force pidlock removal. Be carefull with this option if you have multiple masters.
FORCE_STRANGER_LOCK_RESUME=no FORCE_STRANGER_LOCK_RESUME=no
## Keep partial uploads that can be resumed on next run, experimental feature
PARTIAL=no
## ---------- ALERT OPTIONS ## ---------- ALERT OPTIONS
## List of alert mails separated by spaces ## List of alert mails separated by spaces
DESTINATION_MAILS="your@alert.tld" DESTINATION_MAILS="your@alert.tld"
## Windows (MSYS environment) only mail options (used with sendemail.exe from Brandon Zehm) ## Windows (MSYS environment) only mail options (used by sendemail.exe)
SENDER_MAIL="alert@your.system.tld" SENDER_MAIL="alert@your.system"
SMTP_SERVER=smtp.your.isp.tld SMTP_SERVER=smtp.your.isp.com
SMTP_USER= SMTP_USER=
SMTP_PASSWORD= SMTP_PASSWORD=

124
tests/conf/remote.conf → dev/tests/conf/remote.conf Executable file → Normal file
View File

@ -1,43 +1,35 @@
#!/usr/bin/env bash #!/usr/bin/env bash
###### Osync - Rsync based two way sync engine with fault tolerance ###### osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) ###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
###### Config file rev 2611201401 ###### osync v1.1x / v1.2x config file rev 2016102101
## ---------- GENERAL OPTIONS ## ---------- GENERAL OPTIONS
## Sync job identification ## Sync job identification
SYNC_ID="sync_test" INSTANCE_ID="remote"
## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one. ## Directories to synchronize.
MASTER_SYNC_DIR="master" ## Initiator is the system osync runs on. The initiator directory must be a local path.
SLAVE_SYNC_DIR="ssh://localhost//tmp/osync_tests/remote/slave" INITIATOR_SYNC_DIR="${HOME}/osync-tests/initiator"
#SLAVE_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir2"
## If slave replica is a remote directory, you must specify a RSA key (please use full path). Please see documentation for further information. ## Target is the system osync synchronizes to (can be the same system as the initiator in case of local sync tasks). The target directory can be a local or remote path.
#SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa" #TARGET_SYNC_DIR="${HOME}/osync-tests/target"
TARGET_SYNC_DIR="ssh://root@localhost:49999/${HOME}/osync-tests/target"
## If the target system is remote, you can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local"
## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
SSH_PASSWORD_FILE=""
## Create sync directories if they do not exist ## Create sync directories if they do not exist
CREATE_DIRS=yes CREATE_DIRS=no
## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist) ## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist)
LOGFILE="" LOGFILE=""
## Generate an alert if initiator or target replicas have less free space than given value in KB. Set this to zero to skip disk space tests.
## List of directories to exclude from sync on both sides (rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_EXCLUDE_PATTERN=""
#RSYNC_EXCLUDE_PATTERN="tmp;archives"
## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## This file has to be in the same directory as the config file
## Paths are relative to sync dirs. One element per line.
RSYNC_EXCLUDE_FROM=""
#RSYNC_EXCLUDE_FROM="exclude.list"
## List elements separator char. You may set an alternative separator char for your directories lists above.
PATH_SEPARATOR_CHAR=";"
## Generate an alert if master or slave replicas have less free space than given value in KB.
MINIMUM_SPACE=10240 MINIMUM_SPACE=10240
## Bandwidth limit Kbytes / second. Leave 0 to disable limitation ## Bandwidth limit Kbytes / second. Leave 0 to disable limitation
@ -47,46 +39,84 @@ BANDWIDTH=0
SUDO_EXEC=no SUDO_EXEC=no
## Paranoia option. Don't change this unless you read the documentation. ## Paranoia option. Don't change this unless you read the documentation.
RSYNC_EXECUTABLE=rsync RSYNC_EXECUTABLE=rsync
## Remote rsync executable path. Leave this empty in most cases
RSYNC_REMOTE_PATH=""
## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
RSYNC_PATTERN_FIRST=include
## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_INCLUDE_PATTERN=""
RSYNC_EXCLUDE_PATTERN="*.php"
#RSYNC_EXCLUDE_PATTERN="tmp;archives"
## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## This file has to be in the same directory as the config file
## Paths are relative to sync dirs. One element per line.
RSYNC_INCLUDE_FROM=""
RSYNC_EXCLUDE_FROM=""
#RSYNC_EXCLUDE_FROM="exclude.list"
## List elements separator char. You may set an alternative separator char for your directories lists above.
PATH_SEPARATOR_CHAR=";"
## ---------- REMOTE SYNC OPTIONS ## ---------- REMOTE SYNC OPTIONS
## ssh compression should be used unless your remote connection is good enough (LAN) ## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes SSH_COMPRESSION=yes
## Ignore ssh known hosts. DANGER WILL ROBINSON DANGER ! This can lead to security issues. Only enable this if you know what you're doing.
SSH_IGNORE_KNOWN_HOSTS=no
## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync. ## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync.
REMOTE_HOST_PING=no REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync. ## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync.
## If you use this function, you should set more than one 3rd party host, and be sure you can ping them. ## If you use this function, you should set more than one 3rd party host, and be sure you can ping them.
## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute. ## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute.
REMOTE_3RD_PARTY_HOSTS="" REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
## Remote rsync executable path. Leave this empty in most cases
RSYNC_REMOTE_PATH=""
## ---------- MISC OPTIONS ## ---------- MISC OPTIONS
## Preserve basic linux permissions
PRESERVE_PERMISSIONS=yes
PRESERVE_OWNER=yes
PRESERVE_GROUP=yes
## On MACOS X, does not work and will be ignored
PRESERVE_EXECUTABILITY=yes
## Preserve ACLS. Make sure source and target FS can manage same ACLs or you'll get loads of errors. ## Preserve ACLS. Make sure source and target FS can manage same ACLs or you'll get loads of errors.
PRESERVE_ACL=no PRESERVE_ACL=yes
## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors. ## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors.
PRESERVE_XATTR=no PRESERVE_XATTR=yes
## Transforms symlinks into referent files/dirs ## Transforms symlinks into referent files/dirs
COPY_SYMLINKS=no COPY_SYMLINKS=no
## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root. ## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
KEEP_DIRLINKS=no KEEP_DIRLINKS=no
## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them. ## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
PRESERVE_HARDLINKS=no PRESERVE_HARDLINKS=no
## Do a full checksum on all files that have identical sizes, they are checksummed to see if they actually are identical. This can take a long time.
CHECKSUM=no
## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression. ## Let RSYNC compress file transfers. Do not use this if both initator and target replicas are on local system. Also, do not use this if you already enabled SSH compression.
RSYNC_COMPRESS=yes RSYNC_COMPRESS=yes
## Maximum execution time (in seconds) for sync process. Soft exec time only generates a warning. Hard exec time will generate a warning and stop sync process. ## Maximum execution time (in seconds) for sync process. Set these values zero will disable max execution times.
## Soft exec time only generates a warning. Hard exec time will generate a warning and stop sync process.
SOFT_MAX_EXEC_TIME=7200 SOFT_MAX_EXEC_TIME=7200
HARD_MAX_EXEC_TIME=10600 HARD_MAX_EXEC_TIME=10600
## Log a message every KEEP_LOGGING seconds just to know the task is still alive
KEEP_LOGGING=1801
## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish. ## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish.
MIN_WAIT=60 MIN_WAIT=60
## Maximum time (in seconds) waiting in file monitor / daemon mode. After this time, sync is run.
## Use 0 to wait indefinitely.
MAX_WAIT=7200
## ---------- BACKUP AND DELETION OPTIONS ## ---------- BACKUP AND DELETION OPTIONS
## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups ## Enabling this option will keep a backup of a file on the target replica if it gets updated from the source replica. Backups will be made to .osync_workdir/backups
@ -96,39 +126,48 @@ CONFLICT_BACKUP_MULTIPLE=no
## Osync will clean backup files after a given number of days. Setting this to 0 will disable cleaning and keep backups forever. Warning: This can be very space consuming. ## Osync will clean backup files after a given number of days. Setting this to 0 will disable cleaning and keep backups forever. Warning: This can be very space consuming.
CONFLICT_BACKUP_DAYS=30 CONFLICT_BACKUP_DAYS=30
## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica. ## If the same file exists on both replicas, newer version will be synced. However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner replica.
CONFLICT_PREVALANCE=master CONFLICT_PREVALANCE=initiator
## On deletion propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted ## On deletion propagation to the target replica, a backup of the deleted files can be kept. Deletions will be kept in .osync_workdir/deleted
SOFT_DELETE=yes SOFT_DELETE=yes
## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming. ## Osync will clean deleted files after a given number of days. Setting this to 0 will disable cleaning and keep deleted files forever. Warning: This can be very space consuming.
SOFT_DELETE_DAYS=30 SOFT_DELETE_DAYS=30
## Optional deletion skip on replicas. Valid values are "initiator", "target", or "initiator,target"
SKIP_DELETION=
## ---------- RESUME OPTIONS ## ---------- RESUME OPTIONS
## Try to resume an aborted sync task ## Try to resume an aborted sync task
RESUME_SYNC=yes RESUME_SYNC=yes
## Number maximum resume tries before initiating a fresh sync. ## Number maximum resume tries before initiating a fresh sync.
RESUME_TRY=2 RESUME_TRY=2
## When a pidlock exists on slave replica that does not correspond to master's sync-id, force pidlock removal. Be careful with this option if you have multiple masters. ## When a pidlock exists on slave replica that does not correspond to the initiator's instance-id, force pidlock removal. Be careful with this option if you have multiple initiators.
FORCE_STRANGER_LOCK_RESUME=no FORCE_STRANGER_LOCK_RESUME=no
## Keep partial uploads that can be resumed on next run, experimental feature ## Keep partial uploads that can be resumed on next run, experimental feature
PARTIAL=no PARTIAL=no
## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
DELTA_COPIES=yes
## ---------- ALERT OPTIONS ## ---------- ALERT OPTIONS
## List of alert mails separated by spaces ## List of alert mails separated by spaces
DESTINATION_MAILS="your@alert.tld" DESTINATION_MAILS=""
## Windows (MSYS environment) only mail options (used with sendemail.exe from Brandon Zehm) ## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/)
SENDER_MAIL="alert@your.system.tld" SENDER_MAIL="alert@your.system.tld"
SMTP_SERVER=smtp.your.isp.tld SMTP_SERVER=smtp.your.isp.tld
SMTP_PORT=25
# encryption can be tls, ssl or none
SMTP_ENCRYPTION=none
SMTP_USER= SMTP_USER=
SMTP_PASSWORD= SMTP_PASSWORD=
## ---------- EXECUTION HOOKS ## ---------- EXECUTION HOOKS
## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_SYNC is set). ## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_OPERATION is set).
LOCAL_RUN_BEFORE_CMD="" LOCAL_RUN_BEFORE_CMD=""
LOCAL_RUN_AFTER_CMD="" LOCAL_RUN_AFTER_CMD=""
@ -139,5 +178,8 @@ REMOTE_RUN_AFTER_CMD=""
MAX_EXEC_TIME_PER_CMD_BEFORE=0 MAX_EXEC_TIME_PER_CMD_BEFORE=0
MAX_EXEC_TIME_PER_CMD_AFTER=0 MAX_EXEC_TIME_PER_CMD_AFTER=0
## Stops Osync execution if one of the above commands fail ## Stops osync execution if one of the above commands fail
STOP_ON_CMD_ERROR=yes STOP_ON_CMD_ERROR=yes
## Run local and remote after sync commands even on failure
RUN_AFTER_CMD_ON_ERROR=no

1374
dev/tests/run_tests.sh Executable file

File diff suppressed because it is too large Load Diff

View File

@ -1,10 +1,10 @@
#!/usr/bin/env bash #!/usr/bin/env bash
PROGRAM=osync PROGRAM=osync
PROGRAM_VERSION=1.1 PROGRAM_VERSION=1.1.6-beta
PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016052601 SCRIPT_BUILD=2016082902
## osync / obackup / pmocr / zsnap install script ## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
@ -12,12 +12,12 @@ SCRIPT_BUILD=2016052601
#TODO: silent mode and no stats mode #TODO: silent mode and no stats mode
CONF_DIR=/etc/$PROGRAM CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR=/usr/local/bin BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=/etc/init.d SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora # Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
SERVICE_DIR_SYSTEMD_SYSTEM=/lib/systemd/system SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=/etc/systemd/user SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
## osync specific code ## osync specific code
OSYNC_SERVICE_FILE_INIT="osync-srv" OSYNC_SERVICE_FILE_INIT="osync-srv"
@ -31,8 +31,8 @@ PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM="pmocr-srv.service"
## Generic code ## Generic code
## Default log file ## Default log file
if [ -w /var/log ]; then if [ -w $FAKEROOT/var/log ]; then
LOG_FILE="/var/log/$PROGRAM-install.log" LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM-install.log" LOG_FILE="$HOME/$PROGRAM-install.log"
else else
@ -95,16 +95,16 @@ function SetOSSettings {
*"Darwin"*) *"Darwin"*)
GROUP=admin GROUP=admin
;; ;;
*)
GROUP=root
;;
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
USER="" USER=""
GROUP="" GROUP=""
;; ;;
*)
GROUP=root
;;
esac esac
if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ]); then if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
QuickLogger "Must be run as $USER." QuickLogger "Must be run as $USER."
exit 1 exit 1
fi fi
@ -141,19 +141,19 @@ function CreateConfDir {
function CopyExampleFiles { function CopyExampleFiles {
if [ -f "./sync.conf.example" ]; then if [ -f "./sync.conf.example" ]; then
cp "./sync.conf.example" "/etc/$PROGRAM/sync.conf.example" cp "./sync.conf.example" "$FAKEROOT/etc/$PROGRAM/sync.conf.example"
fi fi
if [ -f "./host_backup.conf.example" ]; then if [ -f "./host_backup.conf.example" ]; then
cp "./host_backup.conf.example" "/etc/$PROGRAM/host_backup.conf.example" cp "./host_backup.conf.example" "$FAKEROOT/etc/$PROGRAM/host_backup.conf.example"
fi fi
if [ -f "./exlude.list.example" ]; then if [ -f "./exlude.list.example" ]; then
cp "./exclude.list.example" "/etc/$PROGRAM" cp "./exclude.list.example" "$FAKEROOT/etc/$PROGRAM"
fi fi
if [ -f "./snapshot.conf.example" ]; then if [ -f "./snapshot.conf.example" ]; then
cp "./snapshot.conf.example" "/etc/$PROGRAM/snapshot.conf.example" cp "./snapshot.conf.example" "$FAKEROOT/etc/$PROGRAM/snapshot.conf.example"
fi fi
} }
@ -184,7 +184,7 @@ function CopyProgram {
QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]." QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]."
else else
chmod 755 "$BIN_DIR/ssh_filter.sh" chmod 755 "$BIN_DIR/ssh_filter.sh"
if ([ "$USER" != "" ] && [ "$GROUP" != "" ]); then if ([ "$USER" != "" ] && [ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
chown $USER:$GROUP "$BIN_DIR/ssh_filter.sh" chown $USER:$GROUP "$BIN_DIR/ssh_filter.sh"
fi fi
QuickLogger "Copied ssh_filter.sh to [$BIN_DIR]." QuickLogger "Copied ssh_filter.sh to [$BIN_DIR]."
@ -282,6 +282,10 @@ do
esac esac
done done
if [ "$FAKEROOT" != "" ]; then
mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
fi
SetOSSettings SetOSSettings
CreateConfDir CreateConfDir
CopyExampleFiles CopyExampleFiles

567
osync.sh
View File

@ -3,11 +3,11 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1 PROGRAM_VERSION=1.1.6-beta
PROGRAM_BUILD=2016072701 PROGRAM_BUILD=2016113001
IS_STABLE=yes IS_STABLE=yes
## FUNC_BUILD=2016071902 ## FUNC_BUILD=2016071902-i
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -24,6 +24,9 @@ KEEP_LOGGING=1801
## Correct output of sort command (language agnostic sorting) ## Correct output of sort command (language agnostic sorting)
export LC_ALL=C export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body # Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -43,14 +46,16 @@ WARN_ALERT=0
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.1
_VERBOSE=0 _VERBOSE=0
else else
SLEEP_TIME=1
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_VERBOSE=1 _VERBOSE=1
fi fi
if [ "$SLEEP_TIME" == "" ]; then
SLEEP_TIME=.1
fi
SCRIPT_PID=$$ SCRIPT_PID=$$
LOCAL_USER=$(whoami) LOCAL_USER=$(whoami)
@ -76,7 +81,7 @@ fi
# Default alert attachment filename # Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$$.last.log"
# Set error exit code if a piped command fails # Set error exit code if a piped command fails
set -o pipefail set -o pipefail
@ -118,14 +123,20 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
ERROR_ALERT=1 ERROR_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "ERROR" ]; then elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value"
ERROR_ALERT=1 ERROR_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
return return
elif [ "$level" == "WARN" ]; then elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value"
WARN_ALERT=1 WARN_ALERT=1
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a su$
echo "1" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID"
return return
elif [ "$level" == "NOTICE" ]; then elif [ "$level" == "NOTICE" ]; then
_Logger "$prefix$value" _Logger "$prefix$value"
@ -166,32 +177,51 @@ function QuickLogger {
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds { function KillChilds {
local pid="${1}" local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" local self="${2:-false}" # Should parent be killed too ?
if children="$(pgrep -P "$pid")"; then # Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
for child in $children; do if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
KillChilds "$child" true Logger "Bogus pid given [$pid]." "CRITICAL"
done return 1
fi fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if kill -0 "$pid" > /dev/null 2>&1; then
if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
Logger "Sending SIGTERM to process [$pid]." "DEBUG" if children="$(pgrep -P "$pid")"; then
kill -s SIGTERM "$pid" if [[ "$pid" == *"$children"* ]]; then
if [ $? != 0 ]; then Logger "Bogus pgrep implementation." "CRITICAL"
sleep 15 children="${children/$pid/}"
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" fi
kill -9 "$pid" for child in $children; do
if [ $? != 0 ]; then KillChilds "$child" true
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" done
return 1 fi
fi fi
fi
return 0 # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
else if [ "$self" == true ]; then
return 0 # We need to check for pid again because it may have disappeared after recursive function call
fi if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
} }
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
@ -285,7 +315,7 @@ function SendAlert {
fi fi
# Windows specific # Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then if [ "$SMTP_ENCRYPTION" != "tls" ] && [ "$SMTP_ENCRYPTION" != "ssl" ] && [ "$SMTP_ENCRYPTION" != "none" ]; then
Logger "Bogus smtp encryption, assuming none." "WARN" Logger "Bogus smtp encryption, assuming none." "WARN"
@ -298,14 +328,14 @@ function SendAlert {
if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\"" auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
fi fi
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string $(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0 return 0
fi fi
fi fi
# Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/) # Windows specific, kept for compatibility (sendemail from http://caspian.dotconf.net/menu/Software/SendEmail/)
if type sendemail > /dev/null 2>&1 ; then if type sendemail > /dev/null 2>&1 ; then
@ -339,7 +369,7 @@ function SendAlert {
# Delete tmp log file # Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then if [ -f "$ALERT_LOG_FILE" ]; then
rm "$ALERT_LOG_FILE" rm -f "$ALERT_LOG_FILE"
fi fi
} }
@ -373,7 +403,7 @@ function SendEmail {
local auth_string= local auth_string=
if [ ! -f "$attachment" ]; then if [ ! -f "$attachment" ]; then
attachment_command="-a $ALERT_LOG_FILE" attachment_command="-a $attachment"
mail_no_attachment=1 mail_no_attachment=1
else else
mail_no_attachment=0 mail_no_attachment=0
@ -424,7 +454,7 @@ function SendEmail {
fi fi
# Windows specific # Windows specific
if type "mailsend.exe" > /dev/null 2>&1 ; then if type "mailsend.exe" > /dev/null 2>&1 ; then
if [ "$sender_email" == "" ]; then if [ "$sender_email" == "" ]; then
Logger "Missing sender email." "ERROR" Logger "Missing sender email." "ERROR"
return 1 return 1
@ -448,14 +478,14 @@ function SendEmail {
if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then if [ "$smtp_user" != "" ] && [ "$smtp_password" != "" ]; then
auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\"" auth_string="-auth -user \"$smtp_user\" -pass \"$smtp_password\""
fi fi
$(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string $(type mailsend.exe) -f "$sender_email" -t "$destination_mails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtp_server" -port "$smtp_port" $encryption_string $auth_string
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
else else
Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
return 0 return 0
fi fi
fi fi
# pfSense specific # pfSense specific
if [ -f /usr/local/bin/mail.php ]; then if [ -f /usr/local/bin/mail.php ]; then
@ -537,7 +567,7 @@ function Spinner {
# obsolete, use StripQuotes # obsolete, use StripQuotes
function SedStripQuotes { function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
} }
# Usage: var=$(StripSingleQuotes "$var") # Usage: var=$(StripSingleQuotes "$var")
@ -563,7 +593,7 @@ function StripQuotes {
function EscapeSpaces { function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }" echo "${string// /\\ }"
} }
function IsNumeric { function IsNumeric {
@ -748,10 +778,10 @@ function WaitForTaskCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
return 0
else else
return 1 Logger "Could not stop task" "ERROR"
fi fi
return 1
fi fi
fi fi
sleep $SLEEP_TIME sleep $SLEEP_TIME
@ -795,10 +825,16 @@ function WaitForCompletion {
KillChilds $pid KillChilds $pid
if [ $? == 0 ]; then if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE" Logger "Task stopped successfully" "NOTICE"
return 0
else else
return 1 Logger "Could not stop task" "ERROR"
fi fi
return 1
#if [ $? == 0 ]; then
# Logger "Task stopped successfully" "NOTICE"
# return 0
#else
# return 1
#fi
fi fi
fi fi
sleep $SLEEP_TIME sleep $SLEEP_TIME
@ -943,7 +979,7 @@ function RsyncPatternsAdd {
local pattern_type="${1}" # exclude or include local pattern_type="${1}" # exclude or include
local pattern="${2}" local pattern="${2}"
local rest= local rest
# Disable globbing so wildcards from exclusions do not get expanded # Disable globbing so wildcards from exclusions do not get expanded
set -f set -f
@ -969,215 +1005,222 @@ function RsyncPatternsAdd {
} }
function RsyncPatternsFromAdd { function RsyncPatternsFromAdd {
local pattern_type="${1}" local pattern_type="${1}"
local pattern_from="${2}" local pattern_from="${2}"
local pattern_from= ## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
fi
## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ -e "$pattern_from" ]; then
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" fi
fi
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
fi
} }
function RsyncPatterns { function RsyncPatterns {
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" fi
fi if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then fi
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then
fi RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then fi
RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" fi
fi # Use default include first for quicksync runs
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" elif [ "$RSYNC_PATTERN_FIRST" == "include" ] || [ "$_QUICK_SYNC" == "2" ]; then
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN"
fi fi
else if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi fi
if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
fi
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
fi
} }
function PreInit { function PreInit {
## SSH compression ## SSH compression
if [ "$SSH_COMPRESSION" != "no" ]; then if [ "$SSH_COMPRESSION" != "no" ]; then
SSH_COMP=-C SSH_COMP=-C
else else
SSH_COMP= SSH_COMP=
fi fi
## Ignore SSH known host verification ## Ignore SSH known host verification
if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then if [ "$SSH_IGNORE_KNOWN_HOSTS" == "yes" ]; then
SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
fi fi
## Support for older config files without RSYNC_EXECUTABLE option ## Support for older config files without RSYNC_EXECUTABLE option
if [ "$RSYNC_EXECUTABLE" == "" ]; then if [ "$RSYNC_EXECUTABLE" == "" ]; then
RSYNC_EXECUTABLE=rsync RSYNC_EXECUTABLE=rsync
fi fi
## Sudo execution option ## Sudo execution option
if [ "$SUDO_EXEC" == "yes" ]; then if [ "$SUDO_EXEC" == "yes" ]; then
if [ "$RSYNC_REMOTE_PATH" != "" ]; then if [ "$RSYNC_REMOTE_PATH" != "" ]; then
RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"
else else
RSYNC_PATH="sudo $RSYNC_EXECUTABLE" RSYNC_PATH="sudo $RSYNC_EXECUTABLE"
fi fi
COMMAND_SUDO="sudo" COMMAND_SUDO="sudo"
else else
if [ "$RSYNC_REMOTE_PATH" != "" ]; then if [ "$RSYNC_REMOTE_PATH" != "" ]; then
RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"
else else
RSYNC_PATH="$RSYNC_EXECUTABLE" RSYNC_PATH="$RSYNC_EXECUTABLE"
fi fi
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set rsync default arguments
RSYNC_ARGS="-rltD" RSYNC_ARGS="-rltD"
RSYNC_ATTR_ARGS="-pgo" RSYNC_ATTR_ARGS="-pgo"
if [ "$_DRYRUN" -eq 1 ]; then if [ "$_DRYRUN" -eq 1 ]; then
RSYNC_DRY_ARG="-n" RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN" DRY_WARNING="/!\ DRY RUN"
else else
RSYNC_DRY_ARG="" RSYNC_DRY_ARG=""
fi fi
if [ "$PRESERVE_ACL" == "yes" ]; then if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi fi
if [ "$PRESERVE_XATTR" == "yes" ]; then if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z" RSYNC_ARGS=$RSYNC_ARGS" -z"
fi fi
if [ "$COPY_SYMLINKS" == "yes" ]; then if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L" RSYNC_ARGS=$RSYNC_ARGS" -L"
fi fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K" RSYNC_ARGS=$RSYNC_ARGS" -K"
fi fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H" RSYNC_ARGS=$RSYNC_ARGS" -H"
fi fi
if [ "$CHECKSUM" == "yes" ]; then if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi fi
if [ "$PARTIAL" == "yes" ]; then if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi fi
if [ "$DELTA_COPIES" != "no" ]; then if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file" RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi fi
## Set compression executable and extension ## Set compression executable and extension
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
if type xz > /dev/null 2>&1 if type xz > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.xz COMPRESSION_EXTENSION=.xz
elif type lzma > /dev/null 2>&1 elif type lzma > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.lzma COMPRESSION_EXTENSION=.lzma
elif type pigz > /dev/null 2>&1 elif type pigz > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.gz COMPRESSION_EXTENSION=.gz
# obackup specific # obackup specific
COMPRESSION_OPTIONS=--rsyncable COMPRESSION_OPTIONS=--rsyncable
elif type gzip > /dev/null 2>&1 elif type gzip > /dev/null 2>&1
then then
COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL"
COMPRESSION_EXTENSION=.gz COMPRESSION_EXTENSION=.gz
# obackup specific # obackup specific
COMPRESSION_OPTIONS=--rsyncable COMPRESSION_OPTIONS=--rsyncable
else else
COMPRESSION_PROGRAM= COMPRESSION_PROGRAM=
COMPRESSION_EXTENSION= COMPRESSION_EXTENSION=
fi fi
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
} }
function PostInit { function PostInit {
# Define remote commands # Define remote commands
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
} }
function InitLocalOSSettings { function InitLocalOSSettings {
## If running under Msys, some commands do not run the same way ## If running under Msys, some commands do not run the same way
## Using mingw version of find instead of windows one ## Using mingw version of find instead of windows one
## Getting running processes is quite different ## Getting running processes is quite different
## Ping command is not the same ## Ping command is not the same
if [ "$LOCAL_OS" == "msys" ]; then if [ "$LOCAL_OS" == "msys" ]; then
FIND_CMD=$(dirname $BASH)/find FIND_CMD=$(dirname $BASH)/find
# PROCESS_TEST_CMD assumes there is a variable $pid # PROCESS_TEST_CMD assumes there is a variable $pid
# Tested on MSYS and cygwin # Tested on MSYS and cygwin
PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid'
PING_CMD='$SYSTEMROOT\system32\ping -n 2' PING_CMD='$SYSTEMROOT\system32\ping -n 2'
else else
FIND_CMD=find FIND_CMD=find
# PROCESS_TEST_CMD assumes there is a variable $pid # PROCESS_TEST_CMD assumes there is a variable $pid
PROCESS_TEST_CMD='ps -p$pid' PROCESS_TEST_CMD='ps -p$pid'
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
STAT_CMD="stat --format %y" STAT_CMD="stat --format %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
} }
function InitRemoteOSSettings { function InitRemoteOSSettings {
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
REMOTE_FIND_CMD=find REMOTE_FIND_CMD=find
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
REMOTE_STAT_CMD="stat -f \"%Sm\"" REMOTE_STAT_CMD="stat -f \"%Sm\""
REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\""
else else
REMOTE_STAT_CMD="stat --format %y" REMOTE_STAT_CMD="stat --format %y"
REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\""
fi fi
} }
@ -1202,7 +1245,15 @@ function TrapStop {
} }
function TrapQuit { function TrapQuit {
local exitcode= local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=1
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=1
fi
if [ $ERROR_ALERT -ne 0 ]; then if [ $ERROR_ALERT -ne 0 ]; then
UnlockReplicas UnlockReplicas
@ -1234,9 +1285,7 @@ function TrapQuit {
exitcode=240 # Special exit code for daemon mode not stopping on warnings exitcode=240 # Special exit code for daemon mode not stopping on warnings
else else
UnlockReplicas UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then RunAfterHook
RunAfterHook
fi
CleanUp CleanUp
Logger "$PROGRAM finished." "NOTICE" Logger "$PROGRAM finished." "NOTICE"
exitcode=0 exitcode=0
@ -1541,7 +1590,7 @@ function _CheckLocksRemote {
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? == 0 ]; then
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID) lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)
else else
@ -1729,9 +1778,8 @@ function _get_file_ctime_mtime_local {
local replica_type="${2}" # Initiator / Target local replica_type="${2}" # Initiator / Target
local file_list="${3}" # Contains list of files to get time attrs local file_list="${3}" # Contains list of files to get time attrs
#cat "$file_list" | xargs -I {} stat -c '%n;%Z;%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list" while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
} }
function _get_file_ctime_mtime_remote { function _get_file_ctime_mtime_remote {
@ -1739,7 +1787,8 @@ function _get_file_ctime_mtime_remote {
local replica_type="${2}" local replica_type="${2}"
local file_list="${3}" local file_list="${3}"
local cmd= local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"' cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG" Logger "CMD: $cmd" "DEBUG"
eval $cmd eval $cmd
@ -1940,18 +1989,19 @@ function _delete_local {
do do
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ "$SOFT_DELETE" != "no" ]; then if [ "$SOFT_DELETE" != "no" ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $replica_dir$files" "NOTICE" Logger "Soft deleting $replica_dir$files" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ -e "$replica_dir$deletion_dir/$files" ]; then if [ -e "$replica_dir$deletion_dir/$files" ]; then
rm -rf "${replica_dir:?}$deletion_dir/$files" rm -rf "${replica_dir:?}$deletion_dir/$files"
fi fi
@ -2006,7 +2056,7 @@ function _delete_remote {
# Additionnaly, we need to copy the deletetion list to the remote state folder # Additionnaly, we need to copy the deletetion list to the remote state folder
esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")" esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$2\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE" eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -2017,12 +2067,12 @@ function _delete_remote {
exit 1 exit 1
fi fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$TARGET_STATE_DIR/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 & $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
## The following lines are executed remotely ## The following lines are executed remotely
function _logger { function _logger {
local value="${1}" # What to log local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE" echo -e "$value" >&2 # Log to STDERR
if [ $_SILENT -eq 0 ]; then if [ $_SILENT -eq 0 ]; then
echo -e "$value" echo -e "$value"
@ -2066,32 +2116,36 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
IFS=$'\r\n' IFS=$'\r\n'
for files in $(cat "$FILE_LIST") for files in $(cat "$FILE_LIST")
do do
Logger "Processing file [$file]." "DEBUG"
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ "$SOFT_DELETE" != "no" ]; then if [ "$SOFT_DELETE" != "no" ]; then
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE" Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
fi fi
Logger "Full path for deletion is [$REPLICA_DIR$DELETE_DIR/$files]." "DEBUG"
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files" $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
fi fi
if [ -e "$$REPLICA_DIR$files" ]; then if [ -e "$REPLICA_DIR$files" ]; then
# In order to keep full path on soft deletion, create parent directories before move # In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")" parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then if [ "$parentdir" != "." ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir" $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir" $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
else else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"1 $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
fi fi
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR" Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
@ -2109,7 +2163,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files" $COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot delete $REPLICA_DIR$files" "ERROR" Logger "Cannot delete $REPLICA_DIR$files" "ERROR"
echo "$files" >> "$TARGET_STATE_DIR/$FAILED_DELETE_LIST" echo "$files" >> "$FAILED_DELETE_LIST"
fi fi
fi fi
fi fi
@ -2125,7 +2179,7 @@ ENDSSH
## Copy back the deleted failed file list ## Copy back the deleted failed file list
esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\"" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE" eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -2203,8 +2257,6 @@ function Sync {
local resume_sync= local resume_sync=
Logger "Starting synchronization task." "NOTICE" Logger "Starting synchronization task." "NOTICE"
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ -f "${INITIATOR[7]}" ] && [ "$RESUME_SYNC" != "no" ]; then if [ -f "${INITIATOR[7]}" ] && [ "$RESUME_SYNC" != "no" ]; then
resume_sync=$(cat "${INITIATOR[7]}") resume_sync=$(cat "${INITIATOR[7]}")
@ -2383,13 +2435,13 @@ function _SoftDeleteLocal {
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
else else
Dummy & Dummy &
fi fi
@ -2417,14 +2469,14 @@ function _SoftDeleteRemote {
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
if [ $_DRYRUN -eq 1 ]; then if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Does not remove anything." "NOTICE" Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
else else
Logger "Removing files older than $change_time days on target replica." "NOTICE" Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -2432,7 +2484,7 @@ function _SoftDeleteRemote {
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
@ -2442,10 +2494,10 @@ function _SoftDeleteRemote {
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR" Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
else else
Logger "Cleanup complete on target replica." "NOTICE" Logger "Cleanup complete on $replica_type replica." "NOTICE"
fi fi
} }
@ -2602,14 +2654,12 @@ function Init {
fi fi
## Add Rsync include / exclude patterns ## Add Rsync include / exclude patterns
if [ $_QUICK_SYNC -lt 2 ]; then RsyncPatterns
RsyncPatterns
fi
## Conflict options ## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[1]}${INITIATOR[4]}\"" INITIATOR_BACKUP="--backup --backup-dir=\"${INITIATOR[4]}\""
TARGET_BACKUP="--backup --backup-dir=\"${TARGET[1]}${TARGET[4]}\"" TARGET_BACKUP="--backup --backup-dir=\"${TARGET[4]}\""
if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ]; then
INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" INITIATOR_BACKUP="$INITIATOR_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" TARGET_BACKUP="$TARGET_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)"
@ -2782,7 +2832,6 @@ do
;; ;;
--initiator=*) --initiator=*)
_QUICK_SYNC=$(($_QUICK_SYNC + 1)) _QUICK_SYNC=$(($_QUICK_SYNC + 1))
no_maxtime=1
INITIATOR_SYNC_DIR=${i##*=} INITIATOR_SYNC_DIR=${i##*=}
opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\"" opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\""
;; ;;
@ -2790,7 +2839,6 @@ do
_QUICK_SYNC=$(($_QUICK_SYNC + 1)) _QUICK_SYNC=$(($_QUICK_SYNC + 1))
TARGET_SYNC_DIR=${i##*=} TARGET_SYNC_DIR=${i##*=}
opts=$opts" --target=\"$TARGET_SYNC_DIR\"" opts=$opts" --target=\"$TARGET_SYNC_DIR\""
no_maxtime=1
;; ;;
--rsakey=*) --rsakey=*)
SSH_RSA_PRIVATE_KEY=${i##*=} SSH_RSA_PRIVATE_KEY=${i##*=}
@ -2853,6 +2901,10 @@ opts="${opts# *}"
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
if [ "$PATH_SEPARATOR_CHAR" == "" ]; then
PATH_SEPARATOR_CHAR=";"
fi
MIN_WAIT=30 MIN_WAIT=30
REMOTE_OPERATION=no REMOTE_OPERATION=no
else else
@ -2898,7 +2950,7 @@ opts="${opts# *}"
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
if [ $no_maxtime -eq 1 ]; then if [ $no_maxtime -eq 1 ] ; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0
fi fi
@ -2909,5 +2961,4 @@ opts="${opts# *}"
if [ $? == 0 ]; then if [ $? == 0 ]; then
SoftDelete SoftDelete
fi fi
RunAfterHook
fi fi

279
packaging/ARCH/mksrcinfo Executable file
View File

@ -0,0 +1,279 @@
#!/bin/bash
shopt -s extglob
array_build() {
local dest=$1 src=$2 i keys values
# it's an error to try to copy a value which doesn't exist.
declare -p "$2" &>/dev/null || return 1
# Build an array of the indicies of the source array.
eval "keys=(\"\${!$2[@]}\")"
# Clear the destination array
eval "$dest=()"
# Read values indirectly via their index. This approach gives us support
# for associative arrays, sparse arrays, and empty strings as elements.
for i in "${keys[@]}"; do
values+=("printf -v '$dest[$i]' %s \"\${$src[$i]}\";")
done
eval "${values[*]}"
}
funcgrep() {
{ declare -f "$1" || declare -f package; } 2>/dev/null | grep -E "$2"
}
# extract_global_var function compatible with bash 4.2
extract_global_var() {
# $1: variable name
# $2: multivalued
# $3: name of output var
local attr=$1 isarray=$2 outputvar=$3 ref
if (( isarray )); then
array_build ref "$attr"
[[ ${ref[@]} ]] && array_build "$outputvar" "$attr"
else
[[ ${!attr} ]] && printf -v "$outputvar" %s "${!attr}"
fi
}
# extract global_var function compatible with bash 4.3+
old_extract_global_var() {
# $1: variable name
# $2: multivalued
# $3: name of output var
local attr=$1 isarray=$2 outputvar=$3
if (( isarray )); then
declare -n ref=$attr
# Still need to use array_build here because we can't handle the scoping
# semantics that would be included with the use of 'declare -n'.
[[ ${ref[@]} ]] && array_build "$outputvar" "$attr"
else
[[ ${!attr} ]] && printf -v "$outputvar" %s "${!attr}"
fi
}
extract_function_var() {
# $1: function name
# $2: variable name
# $3: multivalued
# $4: name of output var
local funcname=$1 attr=$2 isarray=$3 outputvar=$4 attr_regex= decl= r=1
if (( isarray )); then
printf -v attr_regex '^[[:space:]]* %s\+?=\(' "$2"
else
printf -v attr_regex '^[[:space:]]* %s\+?=[^(]' "$2"
fi
while read -r; do
# strip leading whitespace and any usage of declare
decl=${REPLY##*([[:space:]])}
eval "${decl/#$attr/$outputvar}"
# entering this loop at all means we found a match, so notify the caller.
r=0
done < <(funcgrep "$funcname" "$attr_regex")
return $r
}
pkgbuild_get_attribute() {
# $1: package name
# $2: attribute name
# $3: multivalued
# $4: name of output var
local pkgname=$1 attrname=$2 isarray=$3 outputvar=$4
printf -v "$outputvar" %s ''
if [[ $pkgname ]]; then
extract_global_var "$attrname" "$isarray" "$outputvar"
extract_function_var "package_$pkgname" "$attrname" "$isarray" "$outputvar"
else
extract_global_var "$attrname" "$isarray" "$outputvar"
fi
}
srcinfo_open_section() {
printf '%s = %s\n' "$1" "$2"
}
srcinfo_close_section() {
echo
}
srcinfo_write_attr() {
# $1: attr name
# $2: attr values
local attrname=$1 attrvalues=("${@:2}")
# normalize whitespace, strip leading and trailing
attrvalues=("${attrvalues[@]//+([[:space:]])/ }")
attrvalues=("${attrvalues[@]#[[:space:]]}")
attrvalues=("${attrvalues[@]%[[:space:]]}")
printf "\t$attrname = %s\n" "${attrvalues[@]}"
}
pkgbuild_extract_to_srcinfo() {
# $1: pkgname
# $2: attr name
# $3: multivalued
local pkgname=$1 attrname=$2 isarray=$3 outvalue=
if pkgbuild_get_attribute "$pkgname" "$attrname" "$isarray" 'outvalue'; then
srcinfo_write_attr "$attrname" "${outvalue[@]}"
fi
}
srcinfo_write_section_details() {
local attr package_arch a
local multivalued_arch_attrs=(source provides conflicts depends replaces
optdepends makedepends checkdepends
{md5,sha{1,224,256,384,512}}sums)
for attr in "${singlevalued[@]}"; do
pkgbuild_extract_to_srcinfo "$1" "$attr" 0
done
for attr in "${multivalued[@]}"; do
pkgbuild_extract_to_srcinfo "$1" "$attr" 1
done
pkgbuild_get_attribute "$1" 'arch' 1 'package_arch'
for a in "${package_arch[@]}"; do
# 'any' is special. there's no support for, e.g. depends_any.
[[ $a = any ]] && continue
for attr in "${multivalued_arch_attrs[@]}"; do
pkgbuild_extract_to_srcinfo "$1" "${attr}_$a" 1
done
done
}
srcinfo_write_global() {
local singlevalued=(pkgdesc pkgver pkgrel epoch url install changelog)
local multivalued=(arch groups license checkdepends makedepends
depends optdepends provides conflicts replaces
noextract options backup
source {md5,sha{1,224,256,384,512}}sums)
srcinfo_open_section 'pkgbase' "${pkgbase:-$pkgname}"
srcinfo_write_section_details ''
srcinfo_close_section
}
srcinfo_write_package() {
local singlevalued=(pkgdesc url install changelog)
local multivalued=(arch groups license checkdepends depends optdepends
provides conflicts replaces options backup)
srcinfo_open_section 'pkgname' "$1"
srcinfo_write_section_details "$1"
srcinfo_close_section
}
srcinfo_write() {
local pkg
srcinfo_write_global
for pkg in "${pkgname[@]}"; do
srcinfo_write_package "$pkg"
done
}
clear_environment() {
local environ
mapfile -t environ < <(compgen -A variable |
grep -xvF "$(printf '%s\n' "$@")")
# expect that some variables marked read only will complain here
unset -v "${environ[@]}" 2>/dev/null
}
srcinfo_write_from_pkgbuild() {(
clear_environment PATH
shopt -u extglob
. "$1" || exit 1
shopt -s extglob
srcinfo_write
)}
usage() {
printf '%s\n' \
'mksrcinfo v8' \
'' \
'mksrcinfo reads the target PKGBUILD and writes an equivalent .SRCINFO.' \
'Without passing any arguments, mksrcinfo will read from $PWD/PKGBUILD' \
'and write to $PWD/.SRCINFO.' \
'' \
'Usage: mksrcinfo [/path/to/pkgbuild]' \
' -h display this help message and exit' \
' -o <file> write output to <file>'
}
error() {
printf "==> ERROR: $1\n" "${@:2}" >&2
}
die() {
error "$@"
exit 1
}
write_srcinfo_header() {
local timefmt='%a %b %e %H:%M:%S %Z %Y'
# fiddle with the environment to ensure we get the a consistent language and
# timezone.
TZ=UTC LC_TIME=C \
printf "# Generated by mksrcinfo %s\n# %($timefmt)T\n" 'v8' -1
}
srcinfo_dest=$PWD/.SRCINFO
while getopts ':o:h' flag; do
case $flag in
o)
srcinfo_dest=$OPTARG
;;
:)
die "option '-%s' requires an argument" "$OPTARG"
;;
h)
usage
exit 0
;;
\?)
die "invalid option -- '-%s' (use -h for help)" "$OPTARG"
;;
esac
done
shift $(( OPTIND - 1 ))
[[ -f PKGBUILD ]] || die 'PKGBUILD not found in current directory'
# TODO: replace this with 'makepkg --printsrcinfo' once makepkg>=4.3 is released.
{
write_srcinfo_header
srcinfo_write_from_pkgbuild "${1:-$PWD/PKGBUILD}"
} >"$srcinfo_dest"
# vim: set et ts=2 sw=2:

View File

@ -0,0 +1,25 @@
#!/bin/bash
git clone git+ssh://aur@aur.archlinux.org/osync.git osync.aur &&
cd "osync.aur" &&
srcdir="." &&
source "PKGBUILD" &&
url=$(echo -n ${source[0]} | sed 's/git+//g' | sed 's/#.*//g') &&
branch=$(echo -n ${source[0]} | sed 's/.*#branch=//g') &&
git clone -b $branch $url &&
# Get pkgver from current osync
pkgver=$(grep PROGRAM_VERSION= ./osync/osync.sh)
pkgver=${pkgver##*=}
echo $pkgver
sed -i "s/pkgver=.*/pkgver=$(pkgver)/g" "PKGBUILD" &&
../mksrcinfo &&
rm -rf "osync" &&
git add . &&
git commit -m "Updated version" &&
git push origin master &&
cd .. &&
rm -rf "osync.aur" &&
echo "Package updated successfully"

View File

@ -1,345 +0,0 @@
#!/usr/bin/env bash
# Test dir
TMP="/tmp/osync_tests"
# SSH port used for remote tests
SSH_PORT=49999
# Get dir the tests are stored in
TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
cd "$TEST_DIR"
OSYNC_EXECUTABLE="$(dirname $TEST_DIR)//osync.sh"
declare -A sandbox_osync
#sandbox_osync[quickLocal]="--master=master --slave=slave"
#sandbox_osync[quickRemote]="--master=master --slave=ssh://localhost//tmp/osync_tests/quickRemote/slave"
#sandbox_osync[local]="conf/local.conf"
#sandbox_osync[remote]="conf/remote.conf"
oneTimeSetUp()
{
for i in "${!sandbox_osync[@]}"
do
prepareSandbox "$i"
done
}
oneTimeTearDown()
{
rm -rf "$TMP"
}
prepareSandbox()
{
rm -rf "$TMP/$1"
mkdir -p "$TMP/$1"
pushd "$TMP/$1" >/dev/null
mkdir master
mkdir slave
mkdir expected
popd >/dev/null
}
compareSandbox()
{
diff -aurx .osync_workdir master slave
assertEquals 0 $?
diff -aurx .osync_workdir master expected
assertEquals 0 $?
diff -aurx .osync_workdir slave expected
assertEquals 0 $?
}
syncSandbox()
{
$OSYNC_EXECUTABLE ${sandbox_osync[$1]} >/dev/null
assertEquals 0 $?
}
runSandbox()
{
syncSandbox "$1"
compareSandbox
}
joinSandbox()
{
cd "$TMP/$1"
}
### Tests ###
# One empty file
_testOneEmptyFile()
{
joinSandbox "$1"
# Add one empty file
touch "$2/testOneEmpty"
touch expected/testOneEmpty
runSandbox "$1"
# Change one empty file
echo "Test" > "$2/testOneEmpty"
cp "$2/testOneEmpty" expected/testOneEmpty
runSandbox "$1"
# Empty one file
echo -n "" > "$2/testOneEmpty"
cp "$2/testOneEmpty" expected/testOneEmpty
runSandbox "$1"
# Delete one empty file
cp "$2/testOneEmpty" testOneEmpty
rm "$2/testOneEmpty"
rm expected/testOneEmpty
runSandbox "$1"
# Backup check
if [ "$2" == "master" ]
then
diff -aur slave/.osync_workdir/deleted/testOneEmpty testOneEmpty
else
diff -aur master/.osync_workdir/deleted/testOneEmpty testOneEmpty
fi
assertEquals 0 $?
}
testQuickLocalMasterOneEmptyFile()
{
_testOneEmptyFile quickLocal master
}
testQuickLocalSlaveOneEmptyFile()
{
_testOneEmptyFile quickLocal slave
}
testQuickRemoteMasterOneEmptyFile()
{
_testOneEmptyFile quickRemote master
}
testQuickRemoteSlaveOneEmptyFile()
{
_testOneEmptyFile quickRemote slave
}
testLocalMasterOneEmptyFile()
{
_testOneEmptyFile local master
}
testLocalSlaveOneEmptyFile()
{
_testOneEmptyFile local slave
}
testRemoteMasterOneEmptyFile()
{
_testOneEmptyFile remote master
}
testRemoteSlaveOneEmptyFile()
{
_testOneEmptyFile remote slave
}
# One file
_testOneFile()
{
joinSandbox "$1"
# Add one file
echo "Test" > "$2/testOne"
cp "$2/testOne" expected/testOne
runSandbox "$1"
# Change one file
echo "Test2" > "$2/testOne"
cp "$2/testOne" expected/testOne
runSandbox "$1"
# Delete one file
cp "$2/testOne" testOne
rm "$2/testOne"
rm expected/testOne
runSandbox "$1"
# Backup check
if [ "$2" == "master" ]
then
diff -aur slave/.osync_workdir/deleted/testOne testOne
else
diff -aur master/.osync_workdir/deleted/testOne testOne
fi
assertEquals 0 $?
}
testQuickLocalMasterOneFile()
{
_testOneFile quickLocal master
}
testQuickLocalSlaveOneFile()
{
_testOneFile quickLocal slave
}
testQuickRemoteMasterOneFile()
{
_testOneFile quickRemote master
}
testQuickRemoteSlaveOneFile()
{
_testOneFile quickRemote slave
}
testLocalMasterOneFile()
{
_testOneFile local master
}
testLocalSlaveOneFile()
{
_testOneFile local slave
}
testRemoteMasterOneFile()
{
_testOneFile remote master
}
testRemoteSlaveOneFile()
{
_testOneFile remote slave
}
# Distinct
_testDistinct()
{
joinSandbox "$1"
# Generate files in master
for i in testDistinctM1 testDistinctM2 testDistinctM3
do
mkdir "master/$i"
mkdir "expected/$i"
for j in m1 m2 m3 ; do
echo "$i/$j" > "master/$i/$j"
cp "master/$i/$j" "expected/$i/$j"
done
done
# Generate files in slave
for i in testDistinctS1 testDistinctS2 testDistinctS3
do
mkdir "slave/$i"
mkdir "expected/$i"
for j in s1 s2 s3 ; do
echo "$i/$j" > "slave/$i/$j"
cp "slave/$i/$j" "expected/$i/$j"
done
done
# Generate files in same directories for master and slave
for i in testDistinctMS1 testDistinctMS2 testDistinctMS3
do
mkdir "master/$i"
mkdir "slave/$i"
mkdir "expected/$i"
for j in ms1 ms2 ms3 ; do
echo "$i/$j" > "master/$i/m-$j"
cp "master/$i/m-$j" "expected/$i/m-$j"
echo "$i/$j" > "slave/$i/s-$j"
cp "slave/$i/s-$j" "expected/$i/s-$j"
done
done
runSandbox "$1"
}
testQuickLocalDistinct()
{
_testDistinct quickLocal
}
testQuickRemoteDistinct()
{
_testDistinct quickRemote
}
testLocalDistinct()
{
_testDistinct local
}
testRemoteDistinct()
{
_testDistinct remote
}
# Collision
_testCollision()
{
joinSandbox "$1"
# Slave precedence
echo "Test1" > master/testCollision1
echo "Test2" > slave/testCollision1
touch -d "2004-02-29 16:21:41" master/testCollision1
touch -d "2004-02-29 16:21:42" slave/testCollision1
cp slave/testCollision1 expected/testCollision1
cp master/testCollision1 testCollision1
runSandbox "$1"
# Backup check
diff -aur master/.osync_workdir/backups/testCollision1 testCollision1
assertEquals 0 $?
# Master precedence
echo "Test1" > master/testCollision2
echo "Test2" > slave/testCollision2
touch -d "2004-02-29 16:21:42" master/testCollision2
touch -d "2004-02-29 16:21:41" slave/testCollision2
cp master/testCollision2 expected/testCollision2
cp slave/testCollision2 testCollision2
runSandbox "$1"
# Backup check
diff -aur slave/.osync_workdir/backups/testCollision2 testCollision2
assertEquals 0 $?
# ??
# echo "Test1" > master/testCollision3
# echo "Test2" > slave/testCollision3
# touch -d "2004-02-29 16:21:42" master/testCollision3
# touch -d "2004-02-29 16:21:42" slave/testCollision3
# cp slave/testCollision3 expected/testCollision3
# runSandbox "$1"
}
testQuickLocalCollision()
{
_testCollision quickLocal
}
testQuickRemoteCollision()
{
_testCollision quickRemote
}
testLocalCollision()
{
_testCollision local
}
testRemoteCollision()
{
_testCollision remote
}
#suite()
#{
# suite_addTest "testQuickRemoteMasterOneEmptyFile"
#}
. shunit2/shunit2