Initial osync upload
This commit is contained in:
deajan
parent
fac3638ab0
commit
973e032835
|
|
@ -1,7 +1,7 @@
|
|||
osync
|
||||
=====
|
||||
|
||||
A two way sync script based on rsync that merges obackup script fault tolerance with sync logic based on bitpocket
|
||||
A two way sync script based on rsync that merges obackup script fault tolerance with sync logic derived from bitpocket project.
|
||||
|
||||
## About
|
||||
|
||||
|
|
@ -14,7 +14,8 @@ I then decided to merge my obackup codebase with bitpocket's sync core, osync wa
|
|||
|
||||
## Installation
|
||||
|
||||
No public release yet (expect it in mid July)
|
||||
Not even beta ready yet. The whole code is not stable at all.
|
||||
Hopefully will work by the end of July.
|
||||
|
||||
## Author
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
##### Two way sync script
|
||||
##### (C) 2013 by Orsiris "Ozy" de Jong | ozy@badministrateur.com
|
||||
OSYNC_VERSION=0.0 #### Build 1806201301
|
||||
#### Osyc
|
||||
## Rsync based two way sync engine. Master/slave setup.
|
||||
OSYNC_VERSION=0.4
|
||||
OSYNC_BUILD=1507201301
|
||||
|
||||
DEBUG=yes
|
||||
DEBUG=no
|
||||
SCRIPT_PID=$$
|
||||
|
||||
LOCAL_USER=$(whoami)
|
||||
LOCAL_HOST=$(hostname)
|
||||
|
||||
## Flags
|
||||
error_alert=0
|
||||
dryrun=0
|
||||
silent=0
|
||||
## Default log file until config file is loaded
|
||||
LOG_FILE=/var/log/osync.log
|
||||
|
||||
## Working directory on master
|
||||
OSYNC_DIR=".osync_workdir"
|
||||
|
||||
|
||||
## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log won't be unnecessary big.
|
||||
KEEP_LOGGING=1801
|
||||
|
||||
function Log
|
||||
{
|
||||
|
|
@ -41,16 +47,6 @@ function TrapError
|
|||
fi
|
||||
}
|
||||
|
||||
function TrapStop
|
||||
{
|
||||
LogError " /!\ WARNING: Manual exit of sync script. Synchronization may be in inconsistent state."
|
||||
if [ "$DEBUG" == "no" ]
|
||||
then
|
||||
CleanUp
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Spinner
|
||||
{
|
||||
if [ $silent -eq 1 ]
|
||||
|
|
@ -86,19 +82,11 @@ function Spinner
|
|||
esac
|
||||
}
|
||||
|
||||
function Dummy
|
||||
function CleanUp
|
||||
{
|
||||
exit 1;
|
||||
}
|
||||
|
||||
function StripQuotes
|
||||
{
|
||||
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
|
||||
}
|
||||
|
||||
function EscapeSpaces
|
||||
{
|
||||
echo $(echo $1 | sed 's/ /\\ /g')
|
||||
rm -f /dev/shm/osync_config_$SCRIPT_PID
|
||||
rm -f /dev/shm/osync_run_local_$SCRIPT_PID
|
||||
rm -f /dev/shm/osync_run_remote_$SCRIPT_PID
|
||||
}
|
||||
|
||||
function SendAlert
|
||||
|
|
@ -141,11 +129,11 @@ function LoadConfigFile
|
|||
{
|
||||
if [ ! -f "$1" ]
|
||||
then
|
||||
LogError "Cannot load sync configuration file [$1]. Synchronization cannot start."
|
||||
LogError "Cannot load configuration file [$1]. Sync cannot start."
|
||||
return 1
|
||||
elif [[ $1 != *.conf ]]
|
||||
then
|
||||
LogError "Wrong configuration file supplied [$1]. Synchronization cannot start."
|
||||
LogError "Wrong configuration file supplied [$1]. Sync cannot start."
|
||||
else
|
||||
egrep '^#|^[^ ]*=[^;&]*' "$1" > "/dev/shm/osync_config_$SCRIPT_PID"
|
||||
source "/dev/shm/osync_config_$SCRIPT_PID"
|
||||
|
|
@ -154,31 +142,19 @@ function LoadConfigFile
|
|||
|
||||
function CheckEnvironment
|
||||
{
|
||||
sed --version > /dev/null 2>&1
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "GNU coreutils not found (tested for sed --version). Synchronization cannot start."
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
if [ "$REMOTE_SYNC" == "yes" ]
|
||||
then
|
||||
if ! type -p ssh > /dev/null 2>&1
|
||||
then
|
||||
LogError "ssh not present. Cannot start backup."
|
||||
LogError "ssh not present. Cannot start sync."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
if ! type -p rsync > /dev/null 2>&1
|
||||
then
|
||||
LogError "rsync not present. Backup cannot start."
|
||||
LogError "rsync not present. Sync cannot start."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Waits for pid $1 to complete. Will log an alert if $2 seconds exec time exceeded unless $2 equals 0. Will stop task and log alert if $3 seconds exec time exceeded.
|
||||
|
|
@ -211,7 +187,6 @@ function WaitForTaskCompletition
|
|||
done
|
||||
}
|
||||
|
||||
|
||||
## Runs local command $1 and waits for completition in $2 seconds
|
||||
function RunLocalCommand
|
||||
{
|
||||
|
|
@ -257,8 +232,10 @@ function RunRemoteCommand
|
|||
LogError "Running command [$1] failed."
|
||||
fi
|
||||
|
||||
Log "Command output:"
|
||||
Log "$(cat /dev/shm/osync_run_remote_$SCRIPT_PID)"
|
||||
if [ -f /dev/shm/osync_run_remote_$SCRIPT_PID ]
|
||||
then
|
||||
Log "Command output: $(cat /dev/shm/osync_run_remote_$SCRIPT_PID)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -290,20 +267,6 @@ function RunAfterHook
|
|||
|
||||
function SetCompressionOptions
|
||||
{
|
||||
if [ "$COMPRESSION_PROGRAM" == "xz" ] && type -p xz > /dev/null 2>&1
|
||||
then
|
||||
COMPRESSION_EXTENSION=.xz
|
||||
elif [ "$COMPRESSION_PROGRAM" == "lzma" ] && type -p lzma > /dev/null 2>&1
|
||||
then
|
||||
COMPRESSION_EXTENSION=.lzma
|
||||
elif [ "$COMPRESSION_PROGRAM" == "gzip" ] && type -p gzip > /dev/null 2>&1
|
||||
then
|
||||
COMPRESSION_EXTENSION=.gz
|
||||
COMPRESSION_OPTIONS=--rsyncable
|
||||
else
|
||||
COMPRESSION_EXTENSION=
|
||||
fi
|
||||
|
||||
if [ "$SSH_COMPRESSION" == "yes" ]
|
||||
then
|
||||
SSH_COMP=-C
|
||||
|
|
@ -314,72 +277,25 @@ function SetCompressionOptions
|
|||
|
||||
function SetSudoOptions
|
||||
{
|
||||
## Add this to support prior config files without RSYNC_EXECUTABLE option
|
||||
if [ "$RSYNC_EXECUTABLE" == "" ]
|
||||
then
|
||||
RSYNC_EXECUTABLE=rsync
|
||||
fi
|
||||
|
||||
if [ "$SUDO_EXEC" == "yes" ]
|
||||
then
|
||||
RSYNC_PATH="sudo $(which rsync)"
|
||||
RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)"
|
||||
COMMAND_SUDO="sudo"
|
||||
else
|
||||
RSYNC_PATH="$(which rsync)"
|
||||
RSYNC_PATH="$(which $RSYNC_EXECUTABLE)"
|
||||
COMMAND_SUDO=""
|
||||
fi
|
||||
}
|
||||
|
||||
function CreateLocalStorageDirectories
|
||||
{
|
||||
if [ ! -d $LOCAL_FILE_STORAGE ] && [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
mkdir -p $LOCAL_FILE_STORAGE
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckLocalSpace
|
||||
{
|
||||
# Not elegant solution to make df silent on errors
|
||||
df -P $LOCAL_FILE_STORAGE > /dev/shm/osync_local_space_$SCRIPT_PID 2>&1
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LOCAL_SPACE=0
|
||||
else
|
||||
LOCAL_SPACE=$(cat /dev/shm/osync_local_space_$SCRIPT_PID | tail -1 | awk '{print $4}')
|
||||
fi
|
||||
|
||||
if [ $LOCAL_SPACE -eq 0 ]
|
||||
then
|
||||
LogError "Local disk space reported to be 0 Ko. This may also happen if local storage path doesn't exist."
|
||||
elif [ $SYNC_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ]
|
||||
then
|
||||
LogError "Backup size is smaller then expected."
|
||||
elif [ $LOCAL_STORAGE_WARN_MIN_SPACE -gt $LOCAL_SPACE ]
|
||||
then
|
||||
LogError "Local disk space is lower than warning value ($LOCAL_SPACE free Ko)."
|
||||
elif [ $LOCAL_SPACE -lt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ]
|
||||
then
|
||||
LogError "Local disk space may be insufficient (depending on rsync delta and DB compression ratio)."
|
||||
fi
|
||||
Log "Local Space: $LOCAL_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko - Files size: $TOTAL_FILES_SIZE Ko"
|
||||
}
|
||||
|
||||
function CheckTotalExecutionTime
|
||||
{
|
||||
#### Check if max execution time of whole script as been reached
|
||||
if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ]
|
||||
then
|
||||
if [ $soft_alert_total -eq 0 ]
|
||||
then
|
||||
LogError "Max soft execution time of the whole sync exceeded while backing up $BACKUP_TASK."
|
||||
soft_alert_total=1
|
||||
fi
|
||||
if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ]
|
||||
then
|
||||
LogError "Max hard execution time of the whole backup exceeded while backing up $BACKUP_TASK, stopping backup process."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckConnectivityRemoteHost
|
||||
{
|
||||
if [ "$REMOTE_HOST_PING" != "no" ]
|
||||
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ]
|
||||
then
|
||||
ping $REMOTE_HOST -c 2 > /dev/null 2>&1
|
||||
if [ $? != 0 ]
|
||||
|
|
@ -412,3 +328,227 @@ function CheckConnectivity3rdPartyHosts
|
|||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
function CreateDirs
|
||||
{
|
||||
if ! [ -d $MASTER_SYNC_DIR/$OSYNC_DIR ]
|
||||
then
|
||||
mkdir $MASTER_SYNC_DIR/$OSYNC_DIR
|
||||
fi
|
||||
if ! [ -d $STATE_DIR ]
|
||||
then
|
||||
mkdir $STATE_DIR
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function CheckMasterSlaveDirs
|
||||
{
|
||||
if ! [ -d $MASTER_SYNC_DIR ]
|
||||
then
|
||||
LogError "Master directory [$MASTER_SYNC_DIR] does not exist."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! [ -d $SLAVE_SYNC_DIR ]
|
||||
then
|
||||
LogError "Slave directory [$SLAVE_SYNC_DIR] does not exist."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function LockMaster
|
||||
{
|
||||
echo o
|
||||
}
|
||||
|
||||
function LockSlave
|
||||
{
|
||||
echo o
|
||||
}
|
||||
|
||||
function Sync
|
||||
{
|
||||
## decide if local or remote prevalence
|
||||
|
||||
## Lock master dir
|
||||
## Lock slave dir
|
||||
|
||||
Log "Starting synchronization task."
|
||||
|
||||
## Create local file list
|
||||
Log "Creating current master file list"
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-current
|
||||
Log "Creating current slave file list"
|
||||
## Create distant file list
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-current
|
||||
|
||||
## diff local file list and before file list except if before file list is empty >> deleted
|
||||
Log "Creating master deleted file list"
|
||||
if [ -f $STATE_DIR/master-tree-before ]
|
||||
then
|
||||
comm --nocheck-order -23 $STATE_DIR/master-tree-before $STATE_DIR/master-tree-current > $STATE_DIR/master-deleted-list
|
||||
else
|
||||
touch $STATE_DIR/master-deleted-list
|
||||
fi
|
||||
|
||||
## diff local file list and before file list except if before file list is empty >> deleted
|
||||
Log "Creating slave deleted file list"
|
||||
if [ -f $STATE_DIR/slave-tree-before ]
|
||||
then
|
||||
comm --nocheck-order -23 $STATE_DIR/slave-tree-before $STATE_DIR/slave-tree-current > $STATE_DIR/slave-deleted-list
|
||||
else
|
||||
touch $STATE_DIR/slave-deleted-list
|
||||
fi
|
||||
|
||||
|
||||
## update local -> remote except deleted
|
||||
Log "Updating slave replica"
|
||||
rsync -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
|
||||
## update remote -> local except deleted
|
||||
Log "Updating master replica"
|
||||
rsync -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
|
||||
|
||||
Log "Propagating deletitions to slave"
|
||||
rsync -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/master-deleted-list" --exclude='*' $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
|
||||
Log "Propagating deletitions to master"
|
||||
rsync -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/slave-deleted-list" --exclude='*' $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
|
||||
|
||||
## Create local file list
|
||||
Log "Creating new master file list"
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-before
|
||||
Log "Creating new slave file list"
|
||||
## Create distant file list
|
||||
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-before
|
||||
|
||||
Log "Finished synchronization task."
|
||||
|
||||
}
|
||||
|
||||
function SoftDelete
|
||||
{
|
||||
echo softd
|
||||
}
|
||||
|
||||
function Init
|
||||
{
|
||||
# Set error exit code if a piped command fails
|
||||
set -o pipefail
|
||||
set -o errtrace
|
||||
|
||||
trap TrapStop SIGINT SIGQUIT
|
||||
if [ "$DEBUG" == "yes" ]
|
||||
then
|
||||
trap 'TrapError ${LINENO} $?' ERR
|
||||
fi
|
||||
|
||||
LOG_FILE=/var/log/osync_$osync_VERSION-$BACKUP_ID.log
|
||||
MAIL_ALERT_MSG="Warning: Execution of osync instance $OSYNC_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors."
|
||||
|
||||
STATE_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/state"
|
||||
|
||||
## Working directories to keep backups of updated / deleted files
|
||||
MASTER_BACKUP_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/backups"
|
||||
MASTER_DELETE_DIR="$MASTER_SYNC_DIR/$OSYNC_DIR/deleted"
|
||||
SLAVE_BACKUP_DIR="$SLAVE_SYNC_DIR/$OSYNC_DIR/backups"
|
||||
SLAVE_DELETE_DIR="$SLAVE_SYNC_DIR/$OSYNC_DIR/deleted"
|
||||
}
|
||||
|
||||
function DryRun
|
||||
{
|
||||
echo dry
|
||||
}
|
||||
|
||||
function Main
|
||||
{
|
||||
CreateDirs
|
||||
Sync
|
||||
}
|
||||
|
||||
function Usage
|
||||
{
|
||||
echo "Osync $OSYNC_VERSION $OSYNC_BUILD"
|
||||
echo ""
|
||||
echo "usage: osync /path/to/conf.file [--dry] [--silent]"
|
||||
echo ""
|
||||
echo "--dry: will run osync without actuallyv doing anything; just testing"
|
||||
echo "--silent: will run osync without any output to stdout, usefull for cron jobs"
|
||||
exit 128
|
||||
}
|
||||
|
||||
# Comand line argument flags
|
||||
dryrun=0
|
||||
silent=0
|
||||
# Alert flags
|
||||
soft_alert_total=0
|
||||
error_alert=0
|
||||
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
Usage
|
||||
exit
|
||||
fi
|
||||
|
||||
for i in "$@"
|
||||
do
|
||||
case $i in
|
||||
--dry)
|
||||
dryrun=1
|
||||
;;
|
||||
--silent)
|
||||
silent=1
|
||||
;;
|
||||
--help|-h)
|
||||
Usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
CheckEnvironment
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
if [ "$1" != "" ]
|
||||
then
|
||||
LoadConfigFile $1
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
Init
|
||||
DATE=$(date)
|
||||
Log "---------------------------------------------------------"
|
||||
Log "$DATE - Osync v$OSYNC_VERSION script begin."
|
||||
Log "----------------------------------------------------------"
|
||||
CheckMasterSlaveDirs
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
if [ $dryrun -eq 1 ]
|
||||
then
|
||||
DryRun
|
||||
else
|
||||
RunBeforeHook
|
||||
Main
|
||||
RunAfterHook
|
||||
fi
|
||||
CleanUp
|
||||
fi
|
||||
else
|
||||
LogError "Configuration file could not be loaded."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
LogError "No configuration file provided."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
LogError "Environment not suitable to run osync."
|
||||
fi
|
||||
|
||||
if [ $error_alert -ne 0 ]
|
||||
then
|
||||
SendAlert
|
||||
LogError "Osync finished with errros."
|
||||
exit 1
|
||||
else
|
||||
Log "Osync script finished."
|
||||
exit 0
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
#!/bin/bash
|
||||
|
||||
#### Osync multi-master file synchronizer
|
||||
#### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
|
||||
#### Config file rev 1507201301
|
||||
|
||||
## Sync job identification, any string you want
|
||||
SYNC_ID="sync_test"
|
||||
|
||||
## Directories to synchronize
|
||||
MASTER_SYNC_DIR="/home/git/osync/test/dir1"
|
||||
SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
|
||||
|
||||
## Generate an alert if master or slave have lass space than given value in KB.
|
||||
MINIMUM_SPACE=1048576
|
||||
|
||||
## If enabled, synchronization will be processed with sudo command. See documentation
|
||||
SUDO_EXEC=yes
|
||||
## Paranoia option. Don't change this unless you read the documentation and still feel concerned about security issues.
|
||||
RSYNC_EXECUTABLE=rsync
|
||||
|
||||
##Remote options (will sync slave through ssh tunnel, needs RSA key. See documentation for remote sync.
|
||||
REMOTE_SYNC=no
|
||||
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
|
||||
REMOTE_USER=syncmaster
|
||||
REMOTE_HOST=yourhost.local
|
||||
REMOTE_PORT=22
|
||||
SSH_COMPRESSION=yes
|
||||
REMOTE_HOST_PING=yes
|
||||
REMOTE_3RD_PARTY_HOST="www.kernel.org"
|
||||
|
||||
PRESERVE_ACL=yes
|
||||
PRESERVE_XATTR=yes
|
||||
RSYNC_COMPRESS=yes
|
||||
|
||||
|
||||
|
||||
SOFT_MAX_EXEC_TIME=30000
|
||||
HARD_MAX_EXEC_TIME=36000Ԯ
|
||||
|
||||
CONFLICT_WINNER=master
|
||||
CONFLICT_KEEP_BACKUP=yes
|
||||
CONFLICT_KEEP_BACKUP_DAYS=30
|
||||
|
||||
SOFT_DELETE=yes
|
||||
SOFT_DELETE_DAYS=30
|
||||
|
||||
DESTINATION_MAILS="ozy@badministrateur.com"
|
||||
|
||||
|
||||
LOCAL_RUN_BEFORE_CMD=""
|
||||
LOCAL_RUN_AFTER_CMD=""
|
||||
|
||||
REMOTE_RUN_BEFORE_CMD=""
|
||||
REMOTE_RUN_AFTER_CMD=""
|
||||
|
||||
MAX_EXEC_TIME_PER_CMD_BEFORE=0
|
||||
MAX_EXEC_TIME_PER_CMD_AFTER=0
|
||||
Loading…
Reference in New Issue