diff --git a/dev/n_osync.sh b/dev/n_osync.sh index d54e103..634853a 100755 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.2-RC1+dev -PROGRAM_BUILD=2016121502 +PROGRAM_BUILD=2016121901 IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG @@ -210,7 +210,7 @@ function _CheckReplicasLocal { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaPath" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica path [$replicaPath]." "CRITICAL" $retval @@ -261,16 +261,17 @@ function _CheckReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env COMMAND_SUDO="'$COMMAND_SUDO'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env replicaPath="'$replicaPath'" env CREATE_DIRS="'$CREATE_DIRS'" env DF_CMD="'$DF_CMD'" env MINIMUM_SPACE="'$MINIMUM_SPACE'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### IsInteger SUBSET #### include #### HumanToNumeric SUBSET #### include #### RemoteLogger SUBSET #### + function _CheckReplicasRemoteSub { if [ ! -d "$replicaPath" ]; then if [ "$CREATE_DIRS" == "yes" ]; then - $COMMAND_SUDO mkdir -p "$replicaPath" + mkdir -p "$replicaPath" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica path [$replicaPath]." "CRITICAL" $retval @@ -370,7 +371,7 @@ function _HandleLocksLocal { local writeLocks if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + mkdir -p "$replicaStateDir" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -412,7 +413,7 @@ function _HandleLocksLocal { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" retval=$? if [ $retval -ne 0 ]; then Logger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -445,7 +446,7 @@ function _HandleLocksRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env replicaStateDir="'$replicaStateDir'" env initiatorRunningPidsFlat="\"(${initiatorRunningPids[@]})\"" env lockfile="'$lockfile'" env replicaType="'$replicaType'" env overwrite="'$overwrite'" \ -env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env INSTANCE_ID="'$INSTANCE_ID'" env FORCE_STRANGER_LOCK_RESUME="'$FORCE_STRANGER_LOCK_RESUME'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### ArrayContains SUBSET #### @@ -454,7 +455,7 @@ include #### RemoteLogger SUBSET #### function _HandleLocksRemoteSub { if [ ! -d "$replicaStateDir" ]; then - $COMMAND_SUDO mkdir -p "$replicaStateDir" + mkdir -p "$replicaStateDir" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create state dir [$replicaStateDir]." "CRITICAL" $retval @@ -506,7 +507,7 @@ function _HandleLocksRemoteSub { if [ $writeLocks != true ]; then return 1 else - $COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" + echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Could not create lock file on local $replicaType in [$lockfile]." "CRITICAL" $retval @@ -589,7 +590,7 @@ function _UnlockReplicasLocal { local retval if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm "$lockfile" + rm "$lockfile" retval=$? if [ $retval -ne 0 ]; then Logger "Could not unlock local $replicaType replica." "ERROR" $retval @@ -613,9 +614,9 @@ function _UnlockReplicasRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env lockfile="'$lockfile'" env COMMAND_SUDO="'$COMMAND_SUDO'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env lockfile="'$lockfile'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 if [ -f "$lockfile" ]; then - $COMMAND_SUDO rm -f "$lockfile" + rm -f "$lockfile" fi ENDSSH retval=$? @@ -817,7 +818,7 @@ function _getFileCtimeMtimeRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" +env replicaPath="'$replicaPath'" env replicaType="'$replicaType'" env REMOTE_STAT_CTIME_MTIME_CMD="'$REMOTE_STAT_CTIME_MTIME_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD "$replicaPath$file" | sort; done < ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" if [ -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then rm -f ".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -1056,7 +1057,7 @@ function _deleteLocal { fi if [ ! -d "$replicaDir$deletionDir" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$replicaDir$deletionDir" + mkdir -p "$replicaDir$deletionDir" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot create local replica deletion directory in [$replicaDir$deletionDir]." "ERROR" $retval @@ -1159,9 +1160,9 @@ function _deleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" env COMMAND_SUDO="'$COMMAND_SUDO'" \ +env sync_on_changes=$sync_on_changes env _DRYRUN="'$_DRYRUN'" \ env FILE_LIST="'$(EscapeSpaces "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}/$deletionListFromReplica${INITIATOR[$__deletedListFile]}")'" env REPLICA_DIR="'$(EscapeSpaces "$replicaDir")'" env SOFT_DELETE="'$SOFT_DELETE'" \ -env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" 'bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 +env DELETION_DIR="'$(EscapeSpaces "$deletionDir")'" env FAILED_DELETE_LIST="'$failedDeleteList'" env SUCCESS_DELETE_LIST="'$successDeleteList'" $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID.$TSTAMP" 2>&1 include #### DEBUG SUBSET #### include #### TrapError SUBSET #### include #### RemoteLogger SUBSET #### @@ -1174,7 +1175,7 @@ include #### RemoteLogger SUBSET #### previousFile="" if [ ! -d "$REPLICA_DIR$DELETION_DIR" ] && [ $_DRYRUN == false ]; then - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR" + mkdir -p "$REPLICA_DIR$DELETION_DIR" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETION_DIR]." "ERROR" $retval @@ -1189,7 +1190,7 @@ include #### RemoteLogger SUBSET #### if [ "$SOFT_DELETE" != "no" ]; then if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$DELETION_DIR/$files" ] || [ -L "$REPLICA_DIR$DELETION_DIR/$files" ]; then - $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETION_DIR/$files" + rm -rf "$REPLICA_DIR$DELETION_DIR/$files" fi if [ -e "$REPLICA_DIR$files" ] || [ -L "$REPLICA_DIR$files" ]; then @@ -1197,11 +1198,11 @@ include #### RemoteLogger SUBSET #### parentdir="$(dirname "$files")" if [ "$parentdir" != "." ]; then RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR/$parentdir]." "VERBOSE" - $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" + mkdir -p "$REPLICA_DIR$DELETION_DIR/$parentdir" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR/$parentdir" else RemoteLogger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETION_DIR]." "VERBOSE" - $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" + mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETION_DIR" fi retval=$? if [ $retval -ne 0 ]; then @@ -1217,7 +1218,7 @@ include #### RemoteLogger SUBSET #### if [ $_DRYRUN == false ]; then if [ -e "$REPLICA_DIR$files" ] || [ -e "$REPLICA_DIR$files" ]; then RemoteLogger "Deleting [$REPLICA_DIR$files]." "VERBOSE" - $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" + rm -rf "$REPLICA_DIR$files" retval=$? if [ $retval -ne 0 ]; then RemoteLogger "Cannot delete [$REPLICA_DIR$files]." "ERROR" $retval @@ -1667,7 +1668,7 @@ function _SoftDeleteLocal { Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE" fi - $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing file cleanup on $replicaType replica." "ERROR" $retval @@ -1676,7 +1677,7 @@ function _SoftDeleteLocal { Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP)" "VERBOSE" Logger "File cleanup complete on $replicaType replica." "NOTICE" fi - $COMMAND_SUDO $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 + $FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 retval=$? if [ $retval -ne 0 ]; then Logger "Error while executing directory cleanup on $replicaType replica." "ERROR" $retval @@ -1720,13 +1721,13 @@ function _SoftDeleteRemote { $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ -env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env COMAMND_SUDO="'$COMMAND_SUDO'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 +env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPath="'$replicaDeletionPath'" env changeTime="'$changeTime'" env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" 2>&1 # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? - $COMMAND_SUDO $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then '$COMMAND_SUDO' rm -rf "{}"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? else echo "The $replicaType replica dir [$replicaDeletionPath] does not exist. Skipping cleaning of old files" diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 4f4d8be..3bbc3f2 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -3,7 +3,7 @@ #### OFUNCTIONS MINI SUBSET #### _OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2016121501 +_OFUNCTIONS_BUILD=2016121901 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -1556,7 +1556,7 @@ function PreInit { else RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo -E" else if [ "$RSYNC_REMOTE_PATH" != "" ]; then RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE"