WMDE
/
osync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Preparing sync_attrs function

This commit is contained in:
deajan 2016-02-16 22:15:44 +01:00
parent 47fbcfd817
commit 5fc500afc9
4 changed files with 259 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
dev/debug_osync.sh
View File

@ -1,14 +1,14 @@
#!/usr/bin/env bash #!/usr/bin/env bash
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris de Jong" AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-dev PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016021602 PROGRAM_BUILD=2016021602
IS_STABLE=no IS_STABLE=no
FUNC_BUILD=2016021603 FUNC_BUILD=2016021604
## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
@ -258,6 +258,12 @@ function SendAlert {
# </OSYNC SPECIFIC> # </OSYNC SPECIFIC>
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
mail_no_attachment=1
else
mail_no_attachment=0
fi
MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE") MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
if [ $ERROR_ALERT -eq 1 ]; then if [ $ERROR_ALERT -eq 1 ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -267,8 +273,11 @@ function SendAlert {
subject="Alert for $INSTANCE_ID" subject="Alert for $INSTANCE_ID"
fi fi
if [ mail_no_attachment -eq 0 ]; then
attachment_command="-a $ALERT_LOG_FILE"
fi
if type mutt > /dev/null 2>&1 ; then if type mutt > /dev/null 2>&1 ; then
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN" Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN"
else else
@ -278,9 +287,9 @@ function SendAlert {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if $(type -p mail) -V | grep "GNU" > /dev/null; then if [ $mail_no_attachment -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $ALERT_LOG_FILE" attachment_command="-A $ALERT_LOG_FILE"
elif $(type -p mail) -V > /dev/null; then elif [ $mail_no_attachment -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a $ALERT_LOG_FILE" attachment_command="-a $ALERT_LOG_FILE"
else else
attachment_command="" attachment_command=""
@ -1526,11 +1535,11 @@ function delete_list {
} }
function _get_file_attrs_local { function _get_file_attrs_local {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
} }
function _get_file_attrs_remote { function _get_file_attrs_remote {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
} }
function sync_attrs { function sync_attrs {
@ -1541,9 +1550,9 @@ function sync_attrs {
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
else else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" eval "$rsync_cmd"
@ -1554,19 +1563,19 @@ function sync_attrs {
fi fi
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
Logger "Getting file attributes failed. Stopping execution." "CRITICAL" Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi fi
exit $retval exit $retval
else else
cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | grep -Ev "^[^ ]*(c|s|t)[^ ]* " | grep -E "^[^ ]*(p|o|g|a)[^ ]* " | sed -e 's/^[^ ]* //' > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID"
Logger "Getting file attributes on replicas succeded." "NOTICE" Logger "Getting file attributes on replicas succeded." "NOTICE"
return 0
fi fi
_get_file_attrs_local "$INITIATOR_SYNC_DIR" _get_file_attrs_local "$INITIATOR_SYNC_DIR"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_get_file_attrs_remote "$TARGET_SYNC_DIR" _get_file_attrs_remote $(EscapeSpaces "$TARGET_SYNC_DIR")
else else
_get_file_attrs_local "$TARGET_SYNC_DIR" _get_file_attrs_local "$TARGET_SYNC_DIR"
fi fi
@ -1885,7 +1894,7 @@ function deletion_propagation {
fi fi
} }
###### Sync function in 5 steps of each 2 runs (functions above) ###### Sync function in 6 steps (functions above)
###### ######
###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S) ###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S)
###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S) ###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S)
@ -1930,7 +1939,7 @@ function Sync {
################################################################################################################################################# Actual sync begins here ################################################################################################################################################# Actual sync begins here
## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :( ## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :(
if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "initiator-replica-tree.fail" ]; then if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "${SYNC_ACTION[0]}.fail" ]; then
#initiator_tree_current #initiator_tree_current
tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_CURRENT_FILENAME" tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_CURRENT_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -1968,60 +1977,58 @@ function Sync {
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
#TODO write resume step if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
#sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR" #TODO: escapespaces if remote... need to refactor with sync_update sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR"
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ]; then
if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
else
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
fi
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
deletion_propagation target "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ]; then
deletion_propagation initiator "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
else
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
fi
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ]; then
deletion_propagation target "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[7]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[7]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -2030,8 +2037,7 @@ function Sync {
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ]; then
#initiator_tree_after deletion_propagation initiator "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[8]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[8]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -2040,8 +2046,8 @@ function Sync {
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ]; then
#target_tree_after #initiator_tree_after
tree_list "$TARGET_SYNC_DIR" target "$TREE_AFTER_FILENAME" tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[9]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[9]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -2049,9 +2055,19 @@ function Sync {
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[10]}.fail" ]; then
#target_tree_after
tree_list "$TARGET_SYNC_DIR" target "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[10]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[10]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
Logger "Finished synchronization task." "NOTICE" Logger "Finished synchronization task." "NOTICE"
echo "${SYNC_ACTION[10]}" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[11]}" > "$INITIATOR_LAST_ACTION"
echo "0" > "$INITIATOR_RESUME_COUNT" echo "0" > "$INITIATOR_RESUME_COUNT"
} }
@ -2271,12 +2287,13 @@ function Init {
INITIATOR_LAST_ACTION="$INITIATOR_STATE_DIR/last-action-$INSTANCE_ID$dry_suffix" INITIATOR_LAST_ACTION="$INITIATOR_STATE_DIR/last-action-$INSTANCE_ID$dry_suffix"
INITIATOR_RESUME_COUNT="$INITIATOR_STATE_DIR/resume-count-$INSTANCE_ID$dry_suffix" INITIATOR_RESUME_COUNT="$INITIATOR_STATE_DIR/resume-count-$INSTANCE_ID$dry_suffix"
## Sync function actions (0-9) ## Sync function actions (0-10)
SYNC_ACTION=( SYNC_ACTION=(
'initiator-replica-tree' 'initiator-replica-tree'
'target-replica-tree' 'target-replica-tree'
'initiator-deleted-list' 'initiator-deleted-list'
'target-deleted-list' 'target-deleted-list'
'sync-attrs'
'update-initiator-replica' 'update-initiator-replica'
'update-target-replica' 'update-target-replica'
'delete-propagation-target' 'delete-propagation-target'

138
dev/n_osync.sh
View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris de Jong" AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-dev PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016021602 PROGRAM_BUILD=2016021602
@ -620,11 +620,11 @@ function delete_list {
} }
function _get_file_attrs_local { function _get_file_attrs_local {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
} }
function _get_file_attrs_remote { function _get_file_attrs_remote {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
} }
function sync_attrs { function sync_attrs {
@ -635,9 +635,9 @@ function sync_attrs {
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
else else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" eval "$rsync_cmd"
@ -648,19 +648,19 @@ function sync_attrs {
fi fi
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
Logger "Getting file attributes failed. Stopping execution." "CRITICAL" Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi fi
exit $retval exit $retval
else else
cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | grep -Ev "^[^ ]*(c|s|t)[^ ]* " | grep -E "^[^ ]*(p|o|g|a)[^ ]* " | sed -e 's/^[^ ]* //' > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID"
Logger "Getting file attributes on replicas succeded." "NOTICE" Logger "Getting file attributes on replicas succeded." "NOTICE"
return 0
fi fi
_get_file_attrs_local "$INITIATOR_SYNC_DIR" _get_file_attrs_local "$INITIATOR_SYNC_DIR"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_get_file_attrs_remote "$TARGET_SYNC_DIR" _get_file_attrs_remote $(EscapeSpaces "$TARGET_SYNC_DIR")
else else
_get_file_attrs_local "$TARGET_SYNC_DIR" _get_file_attrs_local "$TARGET_SYNC_DIR"
fi fi
@ -979,7 +979,7 @@ function deletion_propagation {
fi fi
} }
###### Sync function in 5 steps of each 2 runs (functions above) ###### Sync function in 6 steps (functions above)
###### ######
###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S) ###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S)
###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S) ###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S)
@ -1024,7 +1024,7 @@ function Sync {
################################################################################################################################################# Actual sync begins here ################################################################################################################################################# Actual sync begins here
## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :( ## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :(
if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "initiator-replica-tree.fail" ]; then if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "${SYNC_ACTION[0]}.fail" ]; then
#initiator_tree_current #initiator_tree_current
tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_CURRENT_FILENAME" tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_CURRENT_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -1062,60 +1062,58 @@ function Sync {
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
#TODO write resume step if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
#sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR" #TODO: escapespaces if remote... need to refactor with sync_update sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR"
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ]; then
if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
else
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
fi
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
deletion_propagation target "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ]; then
deletion_propagation initiator "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
else
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
fi
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ]; then
deletion_propagation target "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[7]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[7]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -1124,8 +1122,7 @@ function Sync {
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ]; then
#initiator_tree_after deletion_propagation initiator "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[8]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[8]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -1134,8 +1131,8 @@ function Sync {
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ]; then
#target_tree_after #initiator_tree_after
tree_list "$TARGET_SYNC_DIR" target "$TREE_AFTER_FILENAME" tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[9]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[9]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -1143,9 +1140,19 @@ function Sync {
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[10]}.fail" ]; then
#target_tree_after
tree_list "$TARGET_SYNC_DIR" target "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[10]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[10]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
Logger "Finished synchronization task." "NOTICE" Logger "Finished synchronization task." "NOTICE"
echo "${SYNC_ACTION[10]}" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[11]}" > "$INITIATOR_LAST_ACTION"
echo "0" > "$INITIATOR_RESUME_COUNT" echo "0" > "$INITIATOR_RESUME_COUNT"
} }
@ -1365,12 +1372,13 @@ function Init {
INITIATOR_LAST_ACTION="$INITIATOR_STATE_DIR/last-action-$INSTANCE_ID$dry_suffix" INITIATOR_LAST_ACTION="$INITIATOR_STATE_DIR/last-action-$INSTANCE_ID$dry_suffix"
INITIATOR_RESUME_COUNT="$INITIATOR_STATE_DIR/resume-count-$INSTANCE_ID$dry_suffix" INITIATOR_RESUME_COUNT="$INITIATOR_STATE_DIR/resume-count-$INSTANCE_ID$dry_suffix"
## Sync function actions (0-9) ## Sync function actions (0-10)
SYNC_ACTION=( SYNC_ACTION=(
'initiator-replica-tree' 'initiator-replica-tree'
'target-replica-tree' 'target-replica-tree'
'initiator-deleted-list' 'initiator-deleted-list'
'target-deleted-list' 'target-deleted-list'
'sync-attrs'
'update-initiator-replica' 'update-initiator-replica'
'update-target-replica' 'update-target-replica'
'delete-propagation-target' 'delete-propagation-target'

19
dev/ofunctions.sh
View File

@ -1,5 +1,5 @@
FUNC_BUILD=2016021603 FUNC_BUILD=2016021604
## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
@ -249,6 +249,12 @@ function SendAlert {
# </OSYNC SPECIFIC> # </OSYNC SPECIFIC>
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
mail_no_attachment=1
else
mail_no_attachment=0
fi
MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE") MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
if [ $ERROR_ALERT -eq 1 ]; then if [ $ERROR_ALERT -eq 1 ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -258,8 +264,11 @@ function SendAlert {
subject="Alert for $INSTANCE_ID" subject="Alert for $INSTANCE_ID"
fi fi
if [ mail_no_attachment -eq 0 ]; then
attachment_command="-a $ALERT_LOG_FILE"
fi
if type mutt > /dev/null 2>&1 ; then if type mutt > /dev/null 2>&1 ; then
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN" Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN"
else else
@ -269,9 +278,9 @@ function SendAlert {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if $(type -p mail) -V | grep "GNU" > /dev/null; then if [ $mail_no_attachment -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $ALERT_LOG_FILE" attachment_command="-A $ALERT_LOG_FILE"
elif $(type -p mail) -V > /dev/null; then elif [ $mail_no_attachment -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a $ALERT_LOG_FILE" attachment_command="-a $ALERT_LOG_FILE"
else else
attachment_command="" attachment_command=""

153
osync.sh
View File

@ -1,14 +1,14 @@
#!/usr/bin/env bash #!/usr/bin/env bash
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris de Jong" AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-dev PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016021602 PROGRAM_BUILD=2016021602
IS_STABLE=no IS_STABLE=no
FUNC_BUILD=2016021603 FUNC_BUILD=2016021604
## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
@ -246,6 +246,12 @@ function SendAlert {
# </OSYNC SPECIFIC> # </OSYNC SPECIFIC>
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
mail_no_attachment=1
else
mail_no_attachment=0
fi
MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE") MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
if [ $ERROR_ALERT -eq 1 ]; then if [ $ERROR_ALERT -eq 1 ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -255,8 +261,11 @@ function SendAlert {
subject="Alert for $INSTANCE_ID" subject="Alert for $INSTANCE_ID"
fi fi
if [ mail_no_attachment -eq 0 ]; then
attachment_command="-a $ALERT_LOG_FILE"
fi
if type mutt > /dev/null 2>&1 ; then if type mutt > /dev/null 2>&1 ; then
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN" Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN"
else else
@ -266,9 +275,9 @@ function SendAlert {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if $(type -p mail) -V | grep "GNU" > /dev/null; then if [ $mail_no_attachment -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $ALERT_LOG_FILE" attachment_command="-A $ALERT_LOG_FILE"
elif $(type -p mail) -V > /dev/null; then elif [ $mail_no_attachment -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a $ALERT_LOG_FILE" attachment_command="-a $ALERT_LOG_FILE"
else else
attachment_command="" attachment_command=""
@ -1412,9 +1421,9 @@ function sync_attrs {
if [ "$REMOTE_OPERATION" == "yes" ]; then if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
else else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
fi fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG" Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" eval "$rsync_cmd"
@ -1425,19 +1434,19 @@ function sync_attrs {
fi fi
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
Logger "Getting file attributes failed. Stopping execution." "CRITICAL" Logger "Getting file attributes failed [$retval]. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi fi
exit $retval exit $retval
else else
cat "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" | grep -Ev "^[^ ]*(c|s|t)[^ ]* " | grep -E "^[^ ]*(p|o|g|a)[^ ]* " | sed -e 's/^[^ ]* //' > "$RUN_DIR/$PROGRAM.$FUNCNAME-cleaned.$SCRIPT_PID"
Logger "Getting file attributes on replicas succeded." "NOTICE" Logger "Getting file attributes on replicas succeded." "NOTICE"
return 0
fi fi
_get_file_attrs_local "$INITIATOR_SYNC_DIR" _get_file_attrs_local "$INITIATOR_SYNC_DIR"
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
_get_file_attrs_remote "$TARGET_SYNC_DIR" _get_file_attrs_remote $(EscapeSpaces "$TARGET_SYNC_DIR")
else else
_get_file_attrs_local "$TARGET_SYNC_DIR" _get_file_attrs_local "$TARGET_SYNC_DIR"
fi fi
@ -1752,7 +1761,7 @@ function deletion_propagation {
fi fi
} }
###### Sync function in 5 steps of each 2 runs (functions above) ###### Sync function in 6 steps (functions above)
###### ######
###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S) ###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S)
###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S) ###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S)
@ -1796,7 +1805,7 @@ function Sync {
################################################################################################################################################# Actual sync begins here ################################################################################################################################################# Actual sync begins here
## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :( ## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :(
if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "initiator-replica-tree.fail" ]; then if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "${SYNC_ACTION[0]}.fail" ]; then
#initiator_tree_current #initiator_tree_current
tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_CURRENT_FILENAME" tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_CURRENT_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -1834,60 +1843,58 @@ function Sync {
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
#TODO write resume step if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
#sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR" #TODO: escapespaces if remote... need to refactor with sync_update sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR"
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ]; then
if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
else
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
fi
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
deletion_propagation target "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[4]}.success" > "$INITIATOR_LAST_ACTION"
else else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[4]}.fail" > "$INITIATOR_LAST_ACTION"
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ]; then
deletion_propagation initiator "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
else
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ]; then
sync_update initiator target "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[6]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[6]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ]; then
sync_update target initiator "$DELETED_LIST_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[5]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[5]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
fi
fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ]; then
deletion_propagation target "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[7]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[7]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -1896,8 +1903,7 @@ function Sync {
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ]; then
#initiator_tree_after deletion_propagation initiator "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME"
tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[8]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[8]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -1906,8 +1912,8 @@ function Sync {
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ]; then if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ]; then
#target_tree_after #initiator_tree_after
tree_list "$TARGET_SYNC_DIR" target "$TREE_AFTER_FILENAME" tree_list "$INITIATOR_SYNC_DIR" initiator "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then if [ $? == 0 ]; then
echo "${SYNC_ACTION[9]}.success" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[9]}.success" > "$INITIATOR_LAST_ACTION"
else else
@ -1915,9 +1921,19 @@ function Sync {
fi fi
resume_sync="resumed" resume_sync="resumed"
fi fi
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[10]}.fail" ]; then
#target_tree_after
tree_list "$TARGET_SYNC_DIR" target "$TREE_AFTER_FILENAME"
if [ $? == 0 ]; then
echo "${SYNC_ACTION[10]}.success" > "$INITIATOR_LAST_ACTION"
else
echo "${SYNC_ACTION[10]}.fail" > "$INITIATOR_LAST_ACTION"
fi
resume_sync="resumed"
fi
Logger "Finished synchronization task." "NOTICE" Logger "Finished synchronization task." "NOTICE"
echo "${SYNC_ACTION[10]}" > "$INITIATOR_LAST_ACTION" echo "${SYNC_ACTION[11]}" > "$INITIATOR_LAST_ACTION"
echo "0" > "$INITIATOR_RESUME_COUNT" echo "0" > "$INITIATOR_RESUME_COUNT"
} }
@ -2133,12 +2149,13 @@ function Init {
INITIATOR_LAST_ACTION="$INITIATOR_STATE_DIR/last-action-$INSTANCE_ID$dry_suffix" INITIATOR_LAST_ACTION="$INITIATOR_STATE_DIR/last-action-$INSTANCE_ID$dry_suffix"
INITIATOR_RESUME_COUNT="$INITIATOR_STATE_DIR/resume-count-$INSTANCE_ID$dry_suffix" INITIATOR_RESUME_COUNT="$INITIATOR_STATE_DIR/resume-count-$INSTANCE_ID$dry_suffix"
## Sync function actions (0-9) ## Sync function actions (0-10)
SYNC_ACTION=( SYNC_ACTION=(
'initiator-replica-tree' 'initiator-replica-tree'
'target-replica-tree' 'target-replica-tree'
'initiator-deleted-list' 'initiator-deleted-list'
'target-deleted-list' 'target-deleted-list'
'sync-attrs'
'update-initiator-replica' 'update-initiator-replica'
'update-target-replica' 'update-target-replica'
'delete-propagation-target' 'delete-propagation-target'