From 77cbcd97d90defec9c4683d3c666bd431e03b0a6 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 4 Jun 2023 19:53:37 +0200 Subject: [PATCH 01/19] foo --- dev/tests/run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 4f9ce17..ae2e091 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -121,7 +121,7 @@ function SetupSSH { echp "ls -alh ${HOME}/.ssh" ls -alh "${HOME}/.ssh" - echo "cat ${HOME}/.ssh.authorized_keys" + echo "cat ${HOME}/.ssh/authorized_keys" cat "${HOME}/.ssh/authorized_keys" echo "###" @@ -333,14 +333,14 @@ function test_SSH { echo "Running SSH test as ${REMOTE_USER}" # SSH_PORT and SSH_USER are set by oneTimeSetup - ssh -i "${REMOTE_USER}/.ssh/${PUBKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" + ssh -i "${REMOTE_USER}/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" if [ $? -ne 0 ]; then echo "SSH test failed" failure=true fi echo "Running SSH test as $(whoami)" - ssh -i "$(whoami)/.ssh/${PUBKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" + ssh -i "$(whoami)/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" if [ $? -ne 0 ]; then echo "SSH test failed" failure=true From 912906ef8cb9f08f9c46ae3085246d07379dd597 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 10:56:21 +0200 Subject: [PATCH 02/19] Revert "foo" This reverts commit 77cbcd97d90defec9c4683d3c666bd431e03b0a6. --- dev/tests/run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index ae2e091..4f9ce17 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -121,7 +121,7 @@ function SetupSSH { echp "ls -alh ${HOME}/.ssh" ls -alh "${HOME}/.ssh" - echo "cat ${HOME}/.ssh/authorized_keys" + echo "cat ${HOME}/.ssh.authorized_keys" cat "${HOME}/.ssh/authorized_keys" echo "###" @@ -333,14 +333,14 @@ function test_SSH { echo "Running SSH test as ${REMOTE_USER}" # SSH_PORT and SSH_USER are set by oneTimeSetup - ssh -i "${REMOTE_USER}/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" + ssh -i "${REMOTE_USER}/.ssh/${PUBKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" if [ $? -ne 0 ]; then echo "SSH test failed" failure=true fi echo "Running SSH test as $(whoami)" - ssh -i "$(whoami)/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" + ssh -i "$(whoami)/.ssh/${PUBKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" if [ $? -ne 0 ]; then echo "SSH test failed" failure=true From f6fd288cfaf9e83dd54f99d6ad253be128c8410b Mon Sep 17 00:00:00 2001 From: Noah Zalev Date: Tue, 4 Jul 2023 14:33:04 -0400 Subject: [PATCH 03/19] Typo fix --- dev/debug_osync.sh | 2 +- dev/ofunctions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 9d37b56..7702c4e 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -504,7 +504,7 @@ function SendAlert { fi if [ $runAlert == true ]; then - subject="Currently runing - $subject" + subject="Currently running - $subject" else subject="Finished run - $subject" fi diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index a1a3235..205803d 100755 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -512,7 +512,7 @@ function SendAlert { fi if [ $runAlert == true ]; then - subject="Currently runing - $subject" + subject="Currently running - $subject" else subject="Finished run - $subject" fi From 737008d6626ccd59f7abba4aa9b95b99858dff24 Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Thu, 7 Mar 2024 18:37:39 -0300 Subject: [PATCH 04/19] Improve README --- README.md | 91 +++++++++++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 06af294..158e6b6 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,22 @@ [![macos tests](https://github.com/deajan/osync/actions/workflows/macos.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/macos.yml/badge.svg) -A two way filesync script running on bash Linux, BSD, Android, MacOSX, Cygwin, MSYS2, Win10 bash and virtually any system supporting bash). +A two way filesync script running on bash Linux, BSD, Android, MacOSX, Cygwin, MSYS2, Win10 bash and virtually any system supporting bash. File synchronization is bidirectional, and can be run manually, as scheduled task, or triggered on file changes in daemon mode. It is a command line tool rsync wrapper with a lot of additional features baked in. -This is a quickstart guide, you can find the full documentation on the author's site. +This is a quickstart guide, you can find the full documentation on the [author's site](http://www.netpower.fr/osync). ## About -osync provides the following capabilities +osync provides the following capabilities: - Local-Local and Local-Remote sync - Fault tolerance with resume scenarios - POSIX ACL and extended attributes synchronization - Full script Time control - Soft deletions and multiple backups handling -- Before / after run command execution +- Before/after run command execution - Email alerts - Logging facility - Directory monitoring @@ -33,22 +33,22 @@ osync provides the following capabilities osync is a stateful synchronizer. This means it's agentless and doesn't have to monitor files for changes. Instead, it compares replica file lists between two runs. A full run takes about 2 seconds on a local-local replication and about 7 seconds on a local-remote replication. Disabling some features file like attributes preservation and disk space checks may speed up execution. -osync uses a initiator / target sync schema. It can sync local to local or local to remote directories. By definition, initiator replica is always a local directory on the system osync runs on. -osync uses pidlocks to prevent multiple concurrent sync processes on/to the same initiator / target replica. +osync uses a initiator/target sync schema. It can sync local to local or local to remote directories. By definition, initiator replica is always a local directory on the system osync runs on. +osync uses pidlocks to prevent multiple concurrent sync processes on/to the same initiator/target replica. You may launch concurrent sync processes on the same system but as long as the replicas to synchronize are different. Multiple osync tasks may be launched sequentially by osync osync-batch tool. -Currently, it has been tested on CentOS 5.x, 6.x, 7.x, AlmaLinux 9, Fedora 22-25, Debian 6-11, Linux Mint 14-18, Ubuntu 12.04-22.04, FreeBSD 8.3-11, Mac OS X, QTS 4.5.1(x86) and pfSense 2.3.x. -Microsoft Windows is supported via MSYS, Cygwin and or via WSL. +Currently, it has been tested on CentOS 5.x, 6.x, 7.x, AlmaLinux 9, Fedora 22-25, Debian 6-11, Linux Mint 14-18, Ubuntu 12.04-22.04, FreeBSD 8.3-11, macOS, QTS 4.5.1(x86) and pfSense 2.3.x. +Microsoft Windows is supported via MSYS, Cygwin and via WSL. Android support works via Termux. -Some users also have successfully used osync on Gentoo and created an openRC init scriptt for it. +Some users also have successfully used osync on Gentoo and created an OpenRC init script for it. ## Installation osync has been designed to not delete any data, but rather make backups of conflictual files or soft deletes. Nevertheless, you should always have a neat backup of your data before trying a new sync tool. -Getting osync via github (remove the -b "stable" if you want latest dev snapshot) +Getting osync via GitHub (remove the -b "stable" if you want latest dev snapshot) $ git clone -b "stable" https://github.com/deajan/osync $ cd osync @@ -56,10 +56,10 @@ Getting osync via github (remove the -b "stable" if you want latest dev snapshot Installer script accepts some parameters for automation. Launch install.sh --help for options. -There is also a RPM file that should fit RHEL/CentOS/Fedora and basically any RPM based distro, see the github release. -Please note that RPM files will install osync to /usr/bin instead of /usr/local/bin in order to enforce good practices. +There is also an RPM file that should fit RHEL/CentOS/Fedora and basically any RPM based distro, see the GitHub release. +Please note that RPM files will install osync to `/usr/bin` instead of `/usr/local/bin` in order to enforce good practices. -osync will install itself to /usr/local/bin and an example configuration file will be installed to /etc/osync +osync will install itself to `/usr/local/bin` and an example configuration file will be installed to `/etc/osync`. osync needs to run with bash shell. Using any other shell will most probably result in errors. If bash is not your default shell, you may invoke it using @@ -70,7 +70,7 @@ On *BSD and BusyBox, be sure to have bash installed. If you can't install osync, you may just copy osync.sh where you needed and run it from there. -Archlinux packages are available at (thanks to Shadowigor, ) +Arch Linux packages are available at (thanks to Shadowigor, ). ## Upgrade from previous configuration files @@ -87,23 +87,23 @@ The script will backup your config file, update it's content and try to connect ## Usage Osync can work with in three flavors: Quick sync mode, configuration file mode, and daemon mode. -While quick sync mode is convenient to do fast syncs between some directories, a configuration file gives much more functionnality. +While quick sync mode is convenient to do fast syncs between some directories, a configuration file gives much more functionality. Please use double quotes as path delimiters. Do not use escaped characters in path names. -## QuickSync example +### QuickSync example # osync.sh --initiator="/path/to/dir1" --target="/path/to/remote dir2" # osync.sh --initiator="/path/to/another dir" --target="ssh://user@host.com:22//path/to/dir2" --rsakey=/home/user/.ssh/id_rsa_private_key_example.com -## Summary mode +### Summary mode -osync may output only file changes and errors with the following +osync will output only file changes and errors with the following: # osync.sh --initiator="/path/to/dir1" --target="/path/to/dir" --summary --errors-only --no-prefix This also works in configuration file mode. -## QuickSync with minimal options +### QuickSync with minimal options In order to run osync the quickest (without transferring file attributes, without softdeletion, without prior space checks and without remote connectivity checks, you may use the following: @@ -111,11 +111,11 @@ In order to run osync the quickest (without transferring file attributes, withou All the settings described here may also be configured in the conf file. -## Running osync with a Configuration file +### Running osync with a configuration file -You'll have to customize the sync.conf file according to your needs. -If you intend to sync a remote directory, osync will need a pair of private / public RSA keys to perform remote SSH connections. -Also, running sync as superuser requires to configure /etc/sudoers file. +You'll have to customize the `sync.conf` file according to your needs. +If you intend to sync a remote directory, osync will need a pair of private/public RSA keys to perform remote SSH connections. +Also, running sync as superuser requires to configure the `/etc/sudoers` file. Please read the documentation about remote sync setups. Once you've customized a sync.conf file, you may run osync with the following test run: @@ -131,17 +131,17 @@ Verbose option will display which files and attrs are actually synchronized and You may mix "--silent" and "--verbose" parameters to output verbose input only in the log files. No-Maxtime option will disable execution time checks, which is usefull for big initial sync tasks that might take long time. Next runs should then only propagate changes and take much less time. -Once you're confident about your first runs, you may add osync as a cron task like the following in /etc/crontab which would run osync every 30 minutes: +Once you're confident about your first runs, you may add osync as a cron task like the following in `/etc/crontab` which would run osync every 30 minutes: */30 * * * * root /usr/local/bin/osync.sh /etc/osync/my_sync.conf --silent -Please note that this syntax works for RedHat / CentOS. On Debian you might want to remove the username (ie root) in order to make the crontab entry work. +Please note that this syntax works for RedHat/CentOS. On Debian you might want to remove the username (i.e. root) in order to make the crontab entry work. -## Batch mode +### Batch mode You may want to sequentially run multiple sync sets between the same servers. In that case, osync-batch.sh is a nice tool that will run every osync conf file, and, if a task fails, run it again if there's still some time left. -The following example will run all .conf files found in /etc/osync, and retry 3 times every configuration that fails, if the whole sequential run took less than 2 hours. +The following example will run all .conf files found in `/etc/osync`, and retry 3 times every configuration that fails, if the whole sequential run took less than 2 hours. # osync-batch.sh --path=/etc/osync --max-retries=3 --max-exec-time=7200 @@ -149,36 +149,30 @@ Having multiple conf files can then be run in a single cron command like 00 00 * * * root /usr/local/bin/osync-batch.sh --path=/etc/osync --silent -## Daemon mode +### Daemon mode Additionaly, you may run osync in monitor mode, which means it will perform a sync upon file operations on initiator replica. -This can be a drawback on functionnality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. +This can be a drawback on functionality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. Target replica changes are only synced when initiator replica changes occur, or when a given amount of time (default 600 seconds) passed without any changes on initiator replica. File monitor mode can also be launched as a daemon with an init script. Please read the documentation for more info. Note that monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions). -BSD, MacOS X and Windows are not yet supported for this operation mode, unless you find a inotify-tools package on these OSes. +BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. # osync.sh /etc/osync/my_sync.conf --on-changes Osync file monitor mode may be run as system service with the osync-srv script. -You may run the install.sh script which should work in most cases or copy the files by hand (osync.sh to /usr/bin/local, sync.conf to /etc/osync, osync-srv to /etc/init.d for initV, osync-srv@.service to /usr/lib/systemd/system for systemd, osync-srv-openrc to /etc/init.d/osync-srv-openrc for OpenRC). +You may run the install.sh script which should work in most cases or copy the files by hand (osync.sh to `/usr/bin/local`, sync.conf to `/etc/osync`, osync-srv to `/etc/init.d` for initV, osync-srv@.service to `/usr/lib/systemd/system` for systemd, osync-srv-openrc to `/etc/init.d/osync-srv-openrc` for OpenRC). -InitV specific instructions: - - Any configuration file found in /etc/osync will create a osync daemon instance when service is launched on initV with: +For InitV (any configuration file found in `/etc/osync` will create an osync daemon instance when service is launched on initV): $ service osync-srv start $ chkconfig osync-srv on -Systemd specific (one service per config file) - - Launch service (one service per config file to launch) with: +For systemd, launch service (one service per config file to launch) with: $ systemctl start osync-srv@configfile.conf $ systemctl enable osync-srv@configfile.conf -OpenRC specific instructions (user contrib) - - Launch service (one service per config file to launch) with: +For OpenRC (user contrib), launch service (one service per config file to launch) with: $ rc-update add osync-srv.configfile default ## Security enhancements @@ -190,21 +184,24 @@ Please read full documentation in order to configure ssh filter. All kind of contribs are welcome. -When submitting a PR, please be sure to modify files in dev directory (dev/n_osync.sh, dev/ofunctions.sh, dev/common_install.sh etc) as most of the main files are generated via merge.sh. +When submitting a PR, please be sure to modify files in dev directory (`dev/n_osync.sh`, `dev/ofunctions.sh`, `dev/common_install.sh etc`) as most of the main files are generated via merge.sh. When testing your contribs, generate files via merge.sh or use bootstrap.sh which generates a temporary version of n_osync.sh with all includes. -Unit tests are run by travis on every PR, but you may also run them manually which adds some tests that travis can't do, via dev/tests/run_tests.sh -SSH port can be changed on the fly via environment variable SSH_PORT, eg: SSH_PORT=2222 dev/tests/run_tests.sh +Unit tests are run by travis on every PR, but you may also run them manually which adds some tests that travis can't do, via `dev/tests/run_tests.sh`. +SSH port can be changed on the fly via environment variable SSH_PORT, e.g.: + + # SSH_PORT=2222 dev/tests/run_tests.sh Consider reading CODING_CONVENTIONS.TXT before submitting a patch. ## Troubleshooting -You may find osync's logs in `/var/log/osync.[INSTANCE_ID].log` (or current directory if /var/log is not writable). +You may find osync's logs in `/var/log/osync.[INSTANCE_ID].log` (or current directory if `/var/log` is not writable). Additionnaly, you can use the --verbose flag see to what actions are going on. -When opening an issue, please post the corresponding log files. Also, you may run osync with _DEBUG option in order to have more precise logs, eg: -_DEBUG=yes ./osync.sh /path/to/conf +When opening an issue, please post the corresponding log files. Also, you may run osync with _DEBUG option in order to have more precise logs, e.g.: + + # _DEBUG=yes ./osync.sh /path/to/conf ## Uninstalling @@ -214,5 +211,5 @@ The installer script also has an uninstall mode that will keep configuration fil ## Author -Feel free to open an issue on github or mail me for support in my spare time :) +Feel free to open an issue on GitHub or mail me for support in my spare time :) Orsiris de Jong | ozy@netpower.fr From f889eb99e89c20104929af14c36a967e8fda8afb Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Thu, 7 Mar 2024 18:49:00 -0300 Subject: [PATCH 05/19] Use "monitor mode" instead of "daemon mode" and move note to note box --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 158e6b6..d5a541b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ A two way filesync script running on bash Linux, BSD, Android, MacOSX, Cygwin, MSYS2, Win10 bash and virtually any system supporting bash. -File synchronization is bidirectional, and can be run manually, as scheduled task, or triggered on file changes in daemon mode. +File synchronization is bidirectional, and can be run manually, as scheduled task, or triggered on file changes in monitor mode. It is a command line tool rsync wrapper with a lot of additional features baked in. This is a quickstart guide, you can find the full documentation on the [author's site](http://www.netpower.fr/osync). @@ -86,7 +86,7 @@ The script will backup your config file, update it's content and try to connect ## Usage -Osync can work with in three flavors: Quick sync mode, configuration file mode, and daemon mode. +Osync can work with in three flavors: Quick sync mode, configuration file mode, and monitor mode. While quick sync mode is convenient to do fast syncs between some directories, a configuration file gives much more functionality. Please use double quotes as path delimiters. Do not use escaped characters in path names. @@ -149,14 +149,15 @@ Having multiple conf files can then be run in a single cron command like 00 00 * * * root /usr/local/bin/osync-batch.sh --path=/etc/osync --silent -### Daemon mode +### Monitor mode + +> [!NOTE] +> Monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions) BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. Additionaly, you may run osync in monitor mode, which means it will perform a sync upon file operations on initiator replica. This can be a drawback on functionality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. Target replica changes are only synced when initiator replica changes occur, or when a given amount of time (default 600 seconds) passed without any changes on initiator replica. File monitor mode can also be launched as a daemon with an init script. Please read the documentation for more info. -Note that monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions). -BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. # osync.sh /etc/osync/my_sync.conf --on-changes From ed2e8438d2b4055d0dd3fa8b427c2ecf07c8fa8f Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Wed, 13 Mar 2024 00:19:22 -0300 Subject: [PATCH 06/19] Fix code blocks in README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d5a541b..fa458eb 100644 --- a/README.md +++ b/README.md @@ -170,10 +170,12 @@ For InitV (any configuration file found in `/etc/osync` will create an osync dae $ chkconfig osync-srv on For systemd, launch service (one service per config file to launch) with: + $ systemctl start osync-srv@configfile.conf $ systemctl enable osync-srv@configfile.conf For OpenRC (user contrib), launch service (one service per config file to launch) with: + $ rc-update add osync-srv.configfile default ## Security enhancements From b3d723ad07d0f2e5eb511de3015b5ee70c199a9a Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Wed, 13 Mar 2024 00:42:50 -0300 Subject: [PATCH 07/19] Transform tested platforms list into a table --- README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d5a541b..f54dbb5 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,23 @@ osync uses pidlocks to prevent multiple concurrent sync processes on/to the same You may launch concurrent sync processes on the same system but as long as the replicas to synchronize are different. Multiple osync tasks may be launched sequentially by osync osync-batch tool. -Currently, it has been tested on CentOS 5.x, 6.x, 7.x, AlmaLinux 9, Fedora 22-25, Debian 6-11, Linux Mint 14-18, Ubuntu 12.04-22.04, FreeBSD 8.3-11, macOS, QTS 4.5.1(x86) and pfSense 2.3.x. -Microsoft Windows is supported via MSYS, Cygwin and via WSL. -Android support works via Termux. +## Tested platforms + +| Operating system | Version | Method(s) (if applicable) | +|------------------|---------------|---------------------------| +| AlmaLinux | 9 | (Not applicable) | +| Android | Not known | Termux | +| CentOS | 5.x, 6.x, 7.x | (Not applicable) | +| Fedora | 22-25 | (Not applicable) | +| FreeBSD | 8.3-11 | (Not applicable) | +| Debian | 6-11 | (Not applicable) | +| Linux Mint | 14-18 | (Not applicable) | +| macOS | Not known | (Not applicable) | +| pfSense | 2.3.x | (Not applicable) | +| QTS (x86) | 4.5.1 | (Not applicable) | +| Ubuntu | 12.04-22.04 | (Not applicable) | +| Windows | 10 | MSYS, Cygwin and WSL | + Some users also have successfully used osync on Gentoo and created an OpenRC init script for it. ## Installation From 88e1f3f4f6f1f5002438bf2284b42efc9450fbfd Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Wed, 13 Mar 2024 10:23:32 -0300 Subject: [PATCH 08/19] Move Termux method to version column --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f54dbb5..84c008d 100644 --- a/README.md +++ b/README.md @@ -40,20 +40,20 @@ Multiple osync tasks may be launched sequentially by osync osync-batch tool. ## Tested platforms -| Operating system | Version | Method(s) (if applicable) | -|------------------|---------------|---------------------------| -| AlmaLinux | 9 | (Not applicable) | -| Android | Not known | Termux | -| CentOS | 5.x, 6.x, 7.x | (Not applicable) | -| Fedora | 22-25 | (Not applicable) | -| FreeBSD | 8.3-11 | (Not applicable) | -| Debian | 6-11 | (Not applicable) | -| Linux Mint | 14-18 | (Not applicable) | -| macOS | Not known | (Not applicable) | -| pfSense | 2.3.x | (Not applicable) | -| QTS (x86) | 4.5.1 | (Not applicable) | -| Ubuntu | 12.04-22.04 | (Not applicable) | -| Windows | 10 | MSYS, Cygwin and WSL | +| Operating system | Version | Method(s) (if applicable) | +|------------------|------------------------|---------------------------| +| AlmaLinux | 9 | (Not applicable) | +| Android | Not known (via Termux) | (Not applicable) | +| CentOS | 5.x, 6.x, 7.x | (Not applicable) | +| Fedora | 22-25 | (Not applicable) | +| FreeBSD | 8.3-11 | (Not applicable) | +| Debian | 6-11 | (Not applicable) | +| Linux Mint | 14-18 | (Not applicable) | +| macOS | Not known | (Not applicable) | +| pfSense | 2.3.x | (Not applicable) | +| QTS (x86) | 4.5.1 | (Not applicable) | +| Ubuntu | 12.04-22.04 | (Not applicable) | +| Windows | 10 | MSYS, Cygwin and WSL | Some users also have successfully used osync on Gentoo and created an OpenRC init script for it. From 2ce2561f99f314ee15d7bfeed84e84a4302f95bb Mon Sep 17 00:00:00 2001 From: Vitor Henrique <87824454+vitorhcl@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:55:48 -0300 Subject: [PATCH 09/19] Improve Usage section --- README.md | 64 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index fa458eb..7a905e6 100644 --- a/README.md +++ b/README.md @@ -86,15 +86,29 @@ The script will backup your config file, update it's content and try to connect ## Usage -Osync can work with in three flavors: Quick sync mode, configuration file mode, and monitor mode. -While quick sync mode is convenient to do fast syncs between some directories, a configuration file gives much more functionality. -Please use double quotes as path delimiters. Do not use escaped characters in path names. +Osync can work in 3 modes: +1. [:rocket: Quick sync mode](#quick-sync-mode) +2. [:gear: Configuration file mode](#configuration-file-mode) +3. [:mag_right: Monitor mode](#monitor-mode) -### QuickSync example +> [!NOTE] +> Please use double quotes as path delimiters. Do not use escaped characters in path names. + +### :rocket: Quick sync mode + +Quick sync mode is convenient to do fast syncs between some directories. However, the [configuration file mode](#configuration-file-mode) gives much more functionality. # osync.sh --initiator="/path/to/dir1" --target="/path/to/remote dir2" # osync.sh --initiator="/path/to/another dir" --target="ssh://user@host.com:22//path/to/dir2" --rsakey=/home/user/.ssh/id_rsa_private_key_example.com +#### Quick sync with minimal options + +In order to run osync the quickest (without transferring file attributes, without softdeletion, without prior space checks and without remote connectivity checks, you may use the following: + + # MINIMUM_SPACE=0 PRESERVE_ACL=no PRESERVE_XATTR=no SOFT_DELETE_DAYS=0 CONFLICT_BACKUP_DAYS=0 REMOTE_HOST_PING=no osync.sh --initiator="/path/to/another dir" --target="ssh://user@host.com:22//path/to/dir2" --rsakey=/home/user/.ssh/id_rsa_private_key_example.com + +All the settings described here may also be configured in the conf file. + ### Summary mode osync will output only file changes and errors with the following: @@ -103,33 +117,30 @@ osync will output only file changes and errors with the following: This also works in configuration file mode. -### QuickSync with minimal options - -In order to run osync the quickest (without transferring file attributes, without softdeletion, without prior space checks and without remote connectivity checks, you may use the following: - - # MINIMUM_SPACE=0 PRESERVE_ACL=no PRESERVE_XATTR=no SOFT_DELETE_DAYS=0 CONFLICT_BACKUP_DAYS=0 REMOTE_HOST_PING=no osync.sh --initiator="/path/to/another dir" --target="ssh://user@host.com:22//path/to/dir2" --rsakey=/home/user/.ssh/id_rsa_private_key_example.com - -All the settings described here may also be configured in the conf file. - -### Running osync with a configuration file +### :gear: Configuration file mode You'll have to customize the `sync.conf` file according to your needs. -If you intend to sync a remote directory, osync will need a pair of private/public RSA keys to perform remote SSH connections. -Also, running sync as superuser requires to configure the `/etc/sudoers` file. -Please read the documentation about remote sync setups. -Once you've customized a sync.conf file, you may run osync with the following test run: + +If you intend to sync a remote directory, osync will need a pair of private/public RSA keys to perform remote SSH connections. Also, running sync as superuser requires to configure the `/etc/sudoers` file. + +> [!TIP] +> Read the [example configuration file](https://github.com/deajan/osync/blob/master/sync.conf.example) for documentation about remote sync setups. + +Once you've customized a `sync.conf` file, you may run osync with the following test run: # osync.sh /path/to/your.conf --dry -If everything went well, you may run the actual configuration with one of the following: +If everything went well, you may run the actual configuration with: # osync.sh /path/to/your.conf - # osync.sh /path/to/your.conf --verbose - # osync.sh /path/to/your.conf --no-maxtime -Verbose option will display which files and attrs are actually synchronized and which files are to be soft deleted / are in conflict. -You may mix "--silent" and "--verbose" parameters to output verbose input only in the log files. -No-Maxtime option will disable execution time checks, which is usefull for big initial sync tasks that might take long time. Next runs should then only propagate changes and take much less time. +To display which files and attrs are actually synchronized and which files are to be soft deleted / are in conflict, use `--verbose` (you may mix it with `--silent` to output verbose input only in the log files): + + # osync.sh /path/to/your.conf --verbose + +Use `--no-maxtime` to disable execution time checks, which is usefull for big initial sync tasks that might take long time. Next runs should then only propagate changes and take much less time: + + # osync.sh /path/to/your.conf --no-maxtime Once you're confident about your first runs, you may add osync as a cron task like the following in `/etc/crontab` which would run osync every 30 minutes: @@ -139,9 +150,10 @@ Please note that this syntax works for RedHat/CentOS. On Debian you might want t ### Batch mode -You may want to sequentially run multiple sync sets between the same servers. In that case, osync-batch.sh is a nice tool that will run every osync conf file, and, if a task fails, +You may want to sequentially run multiple sync sets between the same servers. In that case, `osync-batch.sh` is a nice tool that will run every osync conf file, and, if a task fails, run it again if there's still some time left. -The following example will run all .conf files found in `/etc/osync`, and retry 3 times every configuration that fails, if the whole sequential run took less than 2 hours. + +To run all `.conf` files found in `/etc/osync`, and retry 3 times every configuration that fails if the whole sequential run took less than 2 hours, use: # osync-batch.sh --path=/etc/osync --max-retries=3 --max-exec-time=7200 @@ -149,7 +161,7 @@ Having multiple conf files can then be run in a single cron command like 00 00 * * * root /usr/local/bin/osync-batch.sh --path=/etc/osync --silent -### Monitor mode +### :mag_right: Monitor mode > [!NOTE] > Monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions) BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. From 51e79cb56a219c072a8783d9dc3752816b21cd26 Mon Sep 17 00:00:00 2001 From: Vitor Henrique <87824454+vitorhcl@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:00:26 -0300 Subject: [PATCH 10/19] Fix modes links --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7a905e6..cecff1f 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Osync can work in 3 modes: > [!NOTE] > Please use double quotes as path delimiters. Do not use escaped characters in path names. -### :rocket: Quick sync mode +### :rocket: Quick sync mode Quick sync mode is convenient to do fast syncs between some directories. However, the [configuration file mode](#configuration-file-mode) gives much more functionality. @@ -117,7 +117,7 @@ osync will output only file changes and errors with the following: This also works in configuration file mode. -### :gear: Configuration file mode +### :gear: Configuration file mode You'll have to customize the `sync.conf` file according to your needs. @@ -161,7 +161,7 @@ Having multiple conf files can then be run in a single cron command like 00 00 * * * root /usr/local/bin/osync-batch.sh --path=/etc/osync --silent -### :mag_right: Monitor mode +### :mag_right: Monitor mode > [!NOTE] > Monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions) BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. From 7aeb2e3b69c28b770075eb4fac6f035e709aff70 Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Wed, 13 Mar 2024 11:26:58 -0300 Subject: [PATCH 11/19] Improve Monitor mode section --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cecff1f..7042268 100644 --- a/README.md +++ b/README.md @@ -166,15 +166,20 @@ Having multiple conf files can then be run in a single cron command like > [!NOTE] > Monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions) BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. -Additionaly, you may run osync in monitor mode, which means it will perform a sync upon file operations on initiator replica. -This can be a drawback on functionality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. -Target replica changes are only synced when initiator replica changes occur, or when a given amount of time (default 600 seconds) passed without any changes on initiator replica. +Monitor mode will perform a sync upon file operations on initiator replica. This can be a drawback on functionality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. Target replica changes are only synced when initiator replica changes occur, or when a given amount of time (600 seconds by default) passed without any changes on initiator replica. + File monitor mode can also be launched as a daemon with an init script. Please read the documentation for more info. +To use this mode, use `--on-changes`: + # osync.sh /etc/osync/my_sync.conf --on-changes -Osync file monitor mode may be run as system service with the osync-srv script. -You may run the install.sh script which should work in most cases or copy the files by hand (osync.sh to `/usr/bin/local`, sync.conf to `/etc/osync`, osync-srv to `/etc/init.d` for initV, osync-srv@.service to `/usr/lib/systemd/system` for systemd, osync-srv-openrc to `/etc/init.d/osync-srv-openrc` for OpenRC). +Osync file monitor mode may be run as system service with the osync-srv script. You may run the install.sh script which should work in most cases or copy the files by hand: +- `osync.sh` to `/usr/bin/local` +- `sync.conf` to `/etc/osync` +- For InitV, `osync-srv` to `/etc/init.d` +- For systemd, `osync-srv@.service` to `/usr/lib/systemd/system` +- For OpenRC, `osync-srv-openrc` to `/etc/init.d/osync-srv-openrc` For InitV (any configuration file found in `/etc/osync` will create an osync daemon instance when service is launched on initV): From e34ebfd4cd6f12765a801360208d284b1d540741 Mon Sep 17 00:00:00 2001 From: Vitor Henrique <87824454+vitorhcl@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:27:27 -0300 Subject: [PATCH 12/19] Improve wording of "Monitor mode" section --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7042268..41423ce 100644 --- a/README.md +++ b/README.md @@ -168,13 +168,13 @@ Having multiple conf files can then be run in a single cron command like Monitor mode will perform a sync upon file operations on initiator replica. This can be a drawback on functionality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. Target replica changes are only synced when initiator replica changes occur, or when a given amount of time (600 seconds by default) passed without any changes on initiator replica. -File monitor mode can also be launched as a daemon with an init script. Please read the documentation for more info. +This mode can also be launched as a daemon with an init script. Please read the documentation for more info. To use this mode, use `--on-changes`: # osync.sh /etc/osync/my_sync.conf --on-changes -Osync file monitor mode may be run as system service with the osync-srv script. You may run the install.sh script which should work in most cases or copy the files by hand: +To run this mode as system service with the `osync-srv` script, you can run the `install.sh` script (which should work in most cases) or copy the files by hand: - `osync.sh` to `/usr/bin/local` - `sync.conf` to `/etc/osync` - For InitV, `osync-srv` to `/etc/init.d` From 99d01b2ed0b380d603fb04fc08420d8d831fb156 Mon Sep 17 00:00:00 2001 From: Vitor Henrique <87824454+vitorhcl@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:33:20 -0300 Subject: [PATCH 13/19] Improve monitor mode section --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 41423ce..32cb297 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ Having multiple conf files can then be run in a single cron command like ### :mag_right: Monitor mode > [!NOTE] -> Monitoring changes requires inotifywait command (inotify-tools package for most Linux distributions) BSD, macOS and Windows are not yet supported for this operation mode, unless you find an inotify-tools package on these OSes. +> Monitoring changes requires inotifywait command (`inotify-tools` package for most Linux distributions). BSD, macOS and Windows are not yet supported for this operation mode, unless you find an `inotify-tool` package on these OSes. Monitor mode will perform a sync upon file operations on initiator replica. This can be a drawback on functionality versus scheduled mode because this mode only launches a sync task if there are file modifications on the initiator replica, without being able to monitor the target replica. Target replica changes are only synced when initiator replica changes occur, or when a given amount of time (600 seconds by default) passed without any changes on initiator replica. @@ -174,7 +174,7 @@ To use this mode, use `--on-changes`: # osync.sh /etc/osync/my_sync.conf --on-changes -To run this mode as system service with the `osync-srv` script, you can run the `install.sh` script (which should work in most cases) or copy the files by hand: +To run this mode as a system service with the `osync-srv` script, you can run the `install.sh` script (which should work in most cases) or copy the files by hand: - `osync.sh` to `/usr/bin/local` - `sync.conf` to `/etc/osync` - For InitV, `osync-srv` to `/etc/init.d` From 35afd1af8a464f82246156c37404026f6eb0c22a Mon Sep 17 00:00:00 2001 From: Vitor Henrique Date: Sat, 23 Mar 2024 12:36:05 -0300 Subject: [PATCH 14/19] Transform methods column in footnotes --- README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 84c008d..d1b875d 100644 --- a/README.md +++ b/README.md @@ -40,20 +40,23 @@ Multiple osync tasks may be launched sequentially by osync osync-batch tool. ## Tested platforms -| Operating system | Version | Method(s) (if applicable) | -|------------------|------------------------|---------------------------| -| AlmaLinux | 9 | (Not applicable) | -| Android | Not known (via Termux) | (Not applicable) | -| CentOS | 5.x, 6.x, 7.x | (Not applicable) | -| Fedora | 22-25 | (Not applicable) | -| FreeBSD | 8.3-11 | (Not applicable) | -| Debian | 6-11 | (Not applicable) | -| Linux Mint | 14-18 | (Not applicable) | -| macOS | Not known | (Not applicable) | -| pfSense | 2.3.x | (Not applicable) | -| QTS (x86) | 4.5.1 | (Not applicable) | -| Ubuntu | 12.04-22.04 | (Not applicable) | -| Windows | 10 | MSYS, Cygwin and WSL | +| Operating system | Version | +|------------------|------------------------| +| AlmaLinux | 9 | +| Android\* | Not known | +| CentOS | 5.x, 6.x, 7.x | +| Fedora | 22-25 | +| FreeBSD | 8.3-11 | +| Debian | 6-11 | +| Linux Mint | 14-18 | +| macOS | Not known | +| pfSense | 2.3.x | +| QTS (x86) | 4.5.1 | +| Ubuntu | 12.04-22.04 | +| Windows\*\* | 10 | + +\* via Termux. +\*\* via MSYS, Cygwin and WSL. Some users also have successfully used osync on Gentoo and created an OpenRC init script for it. From 05e594e101c174f7c88edcd91bcd2d0feaecd708 Mon Sep 17 00:00:00 2001 From: new2f7 Date: Wed, 12 Jun 2024 09:10:51 +0200 Subject: [PATCH 15/19] Add missing '$' in 'systemctl start $SERVICE_NAME@instance.conf' --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index f6a3556..678e6e7 100755 --- a/install.sh +++ b/install.sh @@ -834,7 +834,7 @@ function CopyServiceFiles { Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." "NOTICE" - Logger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." "NOTICE" + Logger "Can be activated with [systemctl start $SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." "NOTICE" Logger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." "NOTICE" Logger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]." "NOTICE" elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then From 0d00fa8ab9a36c5ad3c1ba71938d0661d12319f2 Mon Sep 17 00:00:00 2001 From: new2f7 Date: Wed, 12 Jun 2024 09:13:36 +0200 Subject: [PATCH 16/19] Fix spacing in usage help --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index f6a3556..7269038 100755 --- a/install.sh +++ b/install.sh @@ -921,8 +921,8 @@ function RemoveAll { function Usage { echo "Installs $PROGRAM into $BIN_DIR" echo "options:" - echo "--silent Will log and bypass user interaction." - echo "--no-stats Used with --silent in order to refuse sending anonymous install stats." + echo "--silent Will log and bypass user interaction." + echo "--no-stats Used with --silent in order to refuse sending anonymous install stats." echo "--remove Remove the program." echo "--prefix=/path Use prefix to install path." exit 127 From 6d13e3976ab6977da4f03b1fc7a3283145440c9b Mon Sep 17 00:00:00 2001 From: new2f7 Date: Wed, 12 Jun 2024 13:30:49 +0200 Subject: [PATCH 17/19] Update uninstaller to remove all services files --- install.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/install.sh b/install.sh index f6a3556..b941eb5 100755 --- a/install.sh +++ b/install.sh @@ -907,13 +907,22 @@ function RemoveAll { else Logger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it." "NOTICE" fi - RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" - RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" - RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT" - RemoveFile "$TARGET_HELPER_SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" - RemoveFile "$TARGET_HELPER_SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" - RemoveFile "$TARGET_HELPER_SERVICE_DIR_INIT/$SERVICE_FILE_INIT" + # Try to uninstall every possible service file + #if [ $init == "systemd" ]; then + RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" + RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" + RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM" + RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER" + #elif [ $init == "initV" ]; then + RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT" + RemoveFile "$SERVICE_DIR_INIT/$TARGET_HELPER_SERVICE_FILE_INIT" + #elif [ $init == "openrc" ]; then + RemoveFile "$SERVICE_DIR_OPENRC/$SERVICE_FILE_OPENRC" + RemoveFile "$SERVICE_DIR_OPENRC/$TARGET_HELPER_SERVICE_FILE_OPENRC" + #else + #Logger "Can't uninstall from initV, systemd or openRC." "WARN" + #fi Logger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually." "NOTICE" } From b1e08fe947e5e8ceb9d0357e79b11fb5043cfae2 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 4 Sep 2024 15:25:54 +0200 Subject: [PATCH 18/19] Fix typos as per #264 --- dev/ofunctions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 205803d..0f23488 100755 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -82,7 +82,7 @@ else _LOGGER_VERBOSE=true fi -if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console +if [ "$SLEEP_TIME" == "" ]; then # Leave the possibility to set SLEEP_TIME as environment variable when running with bash -x in order to avoid spamming console SLEEP_TIME=.05 fi #### DEBUG SUBSET END #### From 548f3c573050308b9cdd7e3746cf783c054d5a00 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 21 Sep 2024 20:39:26 +0200 Subject: [PATCH 19/19] Fix Spinner must work without env vars --- dev/ofunctions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 0f23488..dfc136e 100755 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -786,7 +786,7 @@ function _PerfProfiler { #__WITH_PARANOIA_DEBUG _OFUNCTIONS_SPINNER="|/-\\" function Spinner { - if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ] || [ "$_SYNC_ON_CHANGES" == "initiator" ] || [ "$_SYNC_ON_CHANGES" == "target" ] ; then + if [ "$_LOGGER_SILENT" == true ] || [ "$_LOGGER_ERR_ONLY" == true ] || [ "$_SYNC_ON_CHANGES" == "initiator" ] || [ "$_SYNC_ON_CHANGES" == "target" ] ; then return 0 else printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"