This commit is contained in:
Orsiris de Jong 2016-04-01 19:29:37 +00:00
commit 38356a21cd
14 changed files with 70 additions and 2807 deletions

View File

@ -13,8 +13,12 @@ KNOWN ISSUES
RECENT CHANGES
--------------
#TODO: clock compare in doc
#TODO: explain why osync is lowband friendly in doc
#TODO: explain why osync daemon process can still exist after quit for 30s
! XX Apr 2016: osync v1.1 released
! Paranoia debug gives incorrect number of arguments in rsyncpatternsadd
! cat vv | ssh -p 49999 -i /root/.ssh/id_rsa_netpower root@netpower.fr cat | xargs -I {} stat -c "%n;%Z;%Y" "{}" | sort > tes
! Updated documentation (+README.md file)
- Redirected ERROR and WARN messages to stderr to systemd catches them into it's journal
!- Added a systemd unit file (needs testing in file monitor mode yet)

View File

@ -8,7 +8,7 @@ Always use the following header
#!/usr/bin/env bash
PROGRAM="program-name" # Long description
AUTHOR="(L) 20XX-20YY by Orsiris \"Ozy\" de Jong"
AUTHOR="(C) 20XX-20YY by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.example.com me@example.com"
PROGRAM_BUILD=YYYYMMDDVV
@ -145,3 +145,12 @@ As obackup and osync share alot of common functions, ofunctions.sh will host all
Dev programs n_osync.sh and n_obackup.sh will source ofunctions.sh
Release programs will still include ofunctions.sh in order to enhance ease of use.
Ofunctions are defined like:
#__FUNC:FunctionName
function FunctionName {
}
#__ENDFUNC
These functions are inserted into code that has placeholders like #__FUNC:FuncName

View File

@ -1,4 +1,4 @@
Copyright (c) 2013-2015, Orsiris "Ozy" de Jong. ozy@netpower.fr
Copyright (c) 2013-2016, Orsiris de Jong. ozy@netpower.fr
All rights reserved.
Redistribution and use in source and binary forms, with or without

View File

@ -127,10 +127,10 @@ You may run the install.sh script which should work in most cases or copy the fi
$ service osync-srv start
$ chkconfig osync-srv on
Systemd specific
Systemd specific (one service per config file)
$ systemctl start osync-srv@configfile
$ systemctl enable osync-srv@configfile
$ systemctl start osync-srv@configfile.conf
$ systemctl enable osync-srv@configfile.conf
Troubleshooting
---------------

2750
dev/de

File diff suppressed because it is too large Load Diff

View File

@ -7,7 +7,7 @@ PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016040101
IS_STABLE=yes
## FUNC_BUILD=2016033105
## FUNC_BUILD=2016040102
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -797,7 +797,7 @@ function __CheckArguments {
local counted_arguments=$((iterate-4))
if [ $counted_arguments -ne $number_of_arguments ]; then
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, see log file." "ERROR"
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR"
Logger "Arguments passed: $arg_list" "ERROR"
fi
fi
@ -806,8 +806,8 @@ function __CheckArguments {
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local pattern="${1}"
local pattern_type="${2}" # exclude or include
local pattern_type="${1}" # exclude or include
local pattern="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local rest=
@ -836,8 +836,8 @@ function RsyncPatternsAdd {
}
function RsyncPatternsFromAdd {
local pattern_from="${1}"
local pattern_type="${2}"
local pattern_type="${1}"
local pattern_from="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pattern_from=
@ -856,22 +856,22 @@ function RsyncPatterns {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
@ -1134,14 +1134,14 @@ function CheckCurrentConfig {
fi
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR)
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR)
for i in "${yes_no_vars[@]}"; do
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY)
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi"
eval "$test"

View File

@ -99,14 +99,14 @@ function CheckCurrentConfig {
fi
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR)
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR)
for i in "${yes_no_vars[@]}"; do
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY)
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi"
eval "$test"

View File

@ -1,4 +1,4 @@
## FUNC_BUILD=2016033105
## FUNC_BUILD=2016040102
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -788,7 +788,7 @@ function __CheckArguments {
local counted_arguments=$((iterate-4))
if [ $counted_arguments -ne $number_of_arguments ]; then
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, see log file." "ERROR"
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR"
Logger "Arguments passed: $arg_list" "ERROR"
fi
fi
@ -797,8 +797,8 @@ function __CheckArguments {
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local pattern="${1}"
local pattern_type="${2}" # exclude or include
local pattern_type="${1}" # exclude or include
local pattern="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local rest=
@ -827,8 +827,8 @@ function RsyncPatternsAdd {
}
function RsyncPatternsFromAdd {
local pattern_from="${1}"
local pattern_type="${2}"
local pattern_type="${1}"
local pattern_from="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pattern_from=
@ -847,22 +847,22 @@ function RsyncPatterns {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"

View File

@ -135,7 +135,7 @@ fi
function Statistics {
local link="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION"
if type wget > /dev/null; then
if type wget -qO- > /dev/null; then
wget $link > /dev/null 2>&1
if [ $? == 0 ]; then
exit 0

View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash
SUBPROGRAM=osync
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016031301

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
#
# osync-srv Two way directory sync daemon
#

View File

@ -7,7 +7,7 @@ PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016040101
IS_STABLE=yes
## FUNC_BUILD=2016033105
## FUNC_BUILD=2016040102
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -741,8 +741,8 @@ function CheckConnectivity3rdPartyHosts {
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local pattern="${1}"
local pattern_type="${2}" # exclude or include
local pattern_type="${1}" # exclude or include
local pattern="${2}"
local rest=
@ -770,8 +770,8 @@ function RsyncPatternsAdd {
}
function RsyncPatternsFromAdd {
local pattern_from="${1}"
local pattern_type="${2}"
local pattern_type="${1}"
local pattern_from="${2}"
local pattern_from=
@ -788,22 +788,22 @@ function RsyncPatternsFromAdd {
function RsyncPatterns {
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM"
fi
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
@ -1060,14 +1060,14 @@ function CheckCurrentConfig {
fi
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR)
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR)
for i in "${yes_no_vars[@]}"; do
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY)
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi"
eval "$test"

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Test dir
TMP="/tmp/osync_tests"

View File

@ -6,7 +6,7 @@ AUTHOR="(C) 2015 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
OLD_PROGRAM_VERSION="1.0x"
NEW_PROGRAM_VERSION="v1.1x"
PROGRAM_BUILD=2016033101
PROGRAM_BUILD=2016033102
function Init {
OSYNC_DIR=".osync_workdir"
@ -311,8 +311,8 @@ function RewriteConfigFiles {
exit 1
fi
echo "Backing up [$config_file] as [$config_file.v1.0x.save]"
cp --preserve "$config_file" "$config_file.v1.0x.save"
echo "Backing up [$config_file] as [$config_file.save]"
cp --preserve "$config_file" "$config_file.save"
if [ $? != 0 ]; then
echo "Cannot backup config file."
exit 1