diff --git a/osync_0.99RC4.lyx b/osync_0.99RC4.lyx new file mode 100644 index 0000000..36d8d3b --- /dev/null +++ b/osync_0.99RC4.lyx @@ -0,0 +1,2468 @@ +#LyX 2.1 created this file. For more info see http://www.lyx.org/ +\lyxformat 474 +\begin_document +\begin_header +\textclass article +\use_default_options true +\maintain_unincluded_children false +\language english +\language_package default +\inputencoding utf8x +\fontencoding global +\font_roman default +\font_sans default +\font_typewriter default +\font_math auto +\font_default_family default +\use_non_tex_fonts true +\font_sc false +\font_osf false +\font_sf_scale 100 +\font_tt_scale 100 +\graphics default +\default_output_format default +\output_sync 0 +\bibtex_command default +\index_command default +\paperfontsize default +\spacing single +\use_hyperref true +\pdf_title "Osync Configuration guide" +\pdf_author "Orsiris " +\pdf_bookmarks true +\pdf_bookmarksnumbered false +\pdf_bookmarksopen false +\pdf_bookmarksopenlevel 1 +\pdf_breaklinks false +\pdf_pdfborder true +\pdf_colorlinks false +\pdf_backref section +\pdf_pdfusetitle true +\papersize a4paper +\use_geometry true +\use_package amsmath 1 +\use_package amssymb 1 +\use_package cancel 0 +\use_package esint 1 +\use_package mathdots 1 +\use_package mathtools 0 +\use_package mhchem 1 +\use_package stackrel 0 +\use_package stmaryrd 0 +\use_package undertilde 0 +\cite_engine basic +\cite_engine_type default +\biblio_style plain +\use_bibtopic false +\use_indices false +\paperorientation portrait +\suppress_date false +\justification true +\use_refstyle 1 +\boxbgcolor #d0d0d0 +\index Index +\shortcut idx +\color #008000 +\end_index +\leftmargin 2cm +\topmargin 2cm +\rightmargin 2cm +\bottommargin 2cm +\headheight 1cm +\headsep 1cm +\footskip 1cm +\secnumdepth 3 +\tocdepth 3 +\paragraph_separation indent +\paragraph_indentation 2em +\quotes_language swedish +\papercolumns 1 +\papersides 1 +\paperpagestyle default +\listings_params "backgroundcolor={\color{white}},basicstyle={\ttfamily},breaklines=true,frame=single" +\bullet 0 0 6 -1 +\tracking_changes false +\output_changes false +\html_math_output 0 +\html_css_as_file 0 +\html_be_strict false +\end_header + +\begin_body + +\begin_layout Title +Osync v0.99 RC4 Documentation +\end_layout + +\begin_layout Author +Orsiris +\begin_inset Quotes eld +\end_inset + +Ozy +\begin_inset Quotes erd +\end_inset + + de Jong +\end_layout + +\begin_layout Date +26 November 2014 +\end_layout + +\begin_layout Standard +\begin_inset CommandInset href +LatexCommand href +name "http://www.netpower.fr/osync" +target "http://www.netpower.fr/osync" + +\end_inset + + +\end_layout + +\begin_layout Standard +\begin_inset CommandInset line +LatexCommand rule +offset "0.5ex" +width "100col%" +height "1pt" + +\end_inset + + +\end_layout + +\begin_layout Standard +\begin_inset CommandInset toc +LatexCommand tableofcontents + +\end_inset + + +\end_layout + +\begin_layout Section +Introduction +\end_layout + +\begin_layout Subsection +Quickstart guide +\end_layout + +\begin_layout Standard +Osync is a command line two way synchronization tool for Linux / BSD / MacOSX + and Windows that tries to be the most reliable possible for automation. +\end_layout + +\begin_layout Standard +A quickstart guide might be found in the README.md file. +\end_layout + +\begin_layout Subsection +Basic synchronization concepts +\end_layout + +\begin_layout Standard +Synchronization is usually found in two flavors, filesystem sync and file + level sync. + While whole filesystem synchronization is generally a good way, it's also + very greedy in network ressources and is not easy to setup. + That's where file level sync comes in handy, where only some directories + need to be synced. +\end_layout + +\begin_layout Standard +Now imagine you're syncing two remote offices of a same company. + If you're syncing a user's home directory or it's roaming profile as a + night task, the next day, the user will find it's roaming profile up to + date at the other office. +\end_layout + +\begin_layout Standard +But what would happen if two users work on the same file in a public folder, + at the same time, on both offices ? Most sync software would stop sync + and ask what to do. + Others might simply deleted the oldest version of the file, even if it + was modified on both sides. +\end_layout + +\begin_layout Standard +Also, what would happen if a user uploads too much new data ? If the link + between both offices cannot handle enough data transfer in a given time, + any other sync task won't be run, and the sync would continue during the + day, when bandwidth is necessary elsewhere. +\end_layout + +\begin_layout Standard +What would happen if a power fault occurs while synchronization is going + on ? +\end_layout + +\begin_layout Subsection +What exactly can do Osync +\end_layout + +\begin_layout Standard +Osync is designed to synchronize two folders on both local and / or remote + systems. +\end_layout + +\begin_layout Standard +It is time controlled, which means you can decide how much time it should + spend on a sync task before stopping it. +\end_layout + +\begin_layout Standard +It can resume failed or stopped sync tasks, or totally restart the sync + task if more earlier runs failed. +\end_layout + +\begin_layout Standard +It can keep an earlier version of a file in case of a conflict. + It can also keep multiple earlier versions. +\end_layout + +\begin_layout Standard +It can soft delete files, meaning if a user deletes a file on a replica, + it will keep a copy of that file on the other replica in a +\begin_inset Quotes sld +\end_inset + +recycle bin +\begin_inset Quotes srd +\end_inset + +. +\end_layout + +\begin_layout Standard +It will automatically clean old files (soft deleted and conflict backed + up ones) after a defined amount of days. +\end_layout + +\begin_layout Standard +It will check local disk space before trying to sync. +\end_layout + +\begin_layout Standard +It will send a warning email including the whole sync process execution + log if a warning or error is triggered. +\end_layout + +\begin_layout Standard +Pre-processing and post-processing commands can be launched on local and + / or remote systems (usefull to launch snapshot, flush or virtual machines + standby scripts). +\end_layout + +\begin_layout Standard +Multiple concurrent instances of osync can be run as long as they don't + sync the same replicas at the same time. +\end_layout + +\begin_layout Standard +Osync can use rsync or ssh tunnel compression to gain bandwidth. + Bandwidth can also be limited. +\end_layout + +\begin_layout Standard +It can run in quicksync mode for the impatient (nothing to configure except + the replica paths), or with a full blown config file. +\end_layout + +\begin_layout Standard +You may run Osync manually, schedule its runs with cron, or have it monitor + a directory as a daemon. +\end_layout + +\begin_layout Standard +Osync has been succesfully tested on RHEL / CentOS 5, CentOS 6, Centos 7, + Debian Linux 6.0.7, Linux Mint 14, FreeBSD 8.3, Mac OS X, and Windows using + MSYS environment. + +\end_layout + +\begin_layout Subsection +How Osync tries to resolve sync issues +\end_layout + +\begin_layout Standard +Let's get back to the example above, where Osync is used to sync two remote + offices with users' home directories. +\end_layout + +\begin_layout Standard +Now imagine a user uploaded 100GB of data, whereas the WAN link can only + handle 60GB of data transfer every night (let's say a regular maintenance + night goes from 10:00 pm to 6:00 am). +\end_layout + +\begin_layout Standard +Now if cron launches Osync every night at 10:00 pm, and Osync is configured + to run for maximum 10 hours, it would stop at 6am, after having transferred + 60GB. +\end_layout + +\begin_layout Standard +Then, on the next day, it would transfer the remaining 40GB from 10:00 pm + to about 3:30am. +\end_layout + +\begin_layout Standard +Also, if you run chained instances of Osync, one per user, you can decide + how much time Osync would spend per user. + So if a user uploads too much data, Osync will keep running for that user + for a given amount of time, and then runs for the next user so everyone + will get it's data synced, regarless if the first user has uploaded too + much data. +\end_layout + +\begin_layout Subsection +Naming in this document +\end_layout + +\begin_layout Standard +Osync's goal is to synchronize two directories, that can be hosted on the + same computer or two different ones. +\end_layout + +\begin_layout Standard +The computer that runs Osync must hold at least one of the two directories, + and will be called the +\emph on +local system. +\end_layout + +\begin_layout Standard +The first directory on the local system is called the +\emph on +master replica +\emph default +. +\end_layout + +\begin_layout Standard +The second directory, called the +\emph on +slave replica +\emph default + can be hosted on the +\emph on +local system +\emph default +, or on another computer which will be called the +\emph on +remote system +\emph default +. + In that case, the +\emph on +local system +\emph default + will connect to the +\emph on +remote system +\emph default + through an ssh tunnel and synchronize both +\emph on +master +\emph default + and +\emph on +slave replicas +\emph default +. +\end_layout + +\begin_layout Section +Prerequisites +\end_layout + +\begin_layout Subsection +General packages +\end_layout + +\begin_layout Standard +The following packages are needed on both local and remote systems. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +rsync coreutils +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +File synchronization +\end_layout + +\begin_layout Standard +File sync tasks don't need any special configurations. + You only have to worry about your sync user having enough filesystem permission +s to read / write on both replicas. +\end_layout + +\begin_layout Standard +A good way is to make your user member of the files' group that has full + permissions. +\end_layout + +\begin_layout Standard +Another way to achieve this is using ACLs if your filesystem supports them. + You can add the following permissions for user "syncuser" on directory + "/home/web". + Setting a default rule will add rights on new files. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# setfacl -mR d:g:r-x,g:r-x /home/web +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Be aware that ACLs are tricky and default group permissions serve as mask + for ACLs. + Make always sure you can read your data with your sync user: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# su syncuser +\end_layout + +\begin_layout Plain Layout + +$ cat /master/replica/test.file +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +\begin_inset CommandInset label +LatexCommand label +name "sub:Performing-superuser-backups" + +\end_inset + +Performing superuser sync +\end_layout + +\begin_layout Standard +Osync may be run as superuser, which should always be avoided by granting + the right permissions to a dedicated sync user. + Nevertheless, there are some cases where running as superuser is necessary + to ensure backups of certain system files. + +\end_layout + +\begin_layout Standard +In order to be able to use the sudo command without having to enter a password, + you’ll need to modify the local and / or remote system to allow the following + commands to be run as superuser: rsync, du, find, mkdir, rm, echo and cat. +\end_layout + +\begin_layout Standard +Edit the file /etc/sudoers and add the following +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/usr/bin/rsync +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/usr/bin/du +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/bin/find +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/bin/mkdir +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/bin/rm +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/bin/mv +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/bin/echo +\end_layout + +\begin_layout Plain Layout + +syncuser ALL= NOPASSWD:/bin/cat +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +You might check the right paths to your commands with the following example: + +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# which rsync +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +You'll also need to disable requiretty in /etc/sudoers by adding the following + line: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +Defaults:backupuser !requiretty +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +You should be aware that there is a minor risk with having rsync command + run as superuser. + A user who can run rsync command as superuser can upload any file he wants + to the system, including a tweaked /etc/sudoers or /etc/passwd file. + Please read chapter +\begin_inset CommandInset ref +LatexCommand ref +reference "sub:Enhancing-remote-backup" + +\end_inset + + to secure your installation. +\end_layout + +\begin_layout Subsection +Remote sync +\end_layout + +\begin_layout Standard +Remote synchronization is done through an SSH tunnel. + To be able to establish such a tunnel without having to enter a password, + you’ll have to generate a pair of private and public RSA keys. +\end_layout + +\begin_layout Standard +The private part is kept by the computer that initiates the connection, + the local system.. + The public part is kept on the remote system. +\end_layout + +\begin_layout Standard +The following steps will be required to generate a ssh key. + Please create a dedicated sync user and log in as that user on the local + system to perform the following actions. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +$ ssh-keygen -t rsa +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +This should create two files named ~/.ssh/id_rsa and ~/.ssh/id_rsa.pub +\end_layout + +\begin_layout Standard +The remote system should also have a dedicated sync user ( +\begin_inset Quotes eld +\end_inset + +syncuser +\begin_inset Quotes erd +\end_inset + + in this example). + Both local and remote users don't need to have the same name. +\end_layout + +\begin_layout Standard +Copy the public part of the RSA pair to the remote system with scp (replace + 22 with your ssh port number if needed). +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +$ scp -p 22 ~/.ssh/id_rsa syncuser@remotesystem.tld:/home/syncuser/.ssh/authorized_ +keys +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Make sure the file is readable by the syncuser on the remote system. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# chmod 600 /home/syncuser/.ssh/authorized_keys +\end_layout + +\begin_layout Plain Layout + +# chown syncuser:root /home/syncuser/.ssh/authorized_keys +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Now you should be able to login as "syncuser" on the remote system without + any password. + You can try to remotely login by entering the following on the local system: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +$ ssh -p 22 syncuser@remotesystem.tld +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Be aware that only the user that generated the ssh key can remotely log + in. +\end_layout + +\begin_layout Standard +You may optionnaly enhance remote login security by applying chapter +\begin_inset CommandInset ref +LatexCommand ref +reference "sub:Enhancing-remote-backup" + +\end_inset + + methods. +\end_layout + +\begin_layout Subsection +Mail transport agent +\end_layout + +\begin_layout Standard +You should make sure your system can send emails so Osync can warn you if + something bad happens. + Osync will use mutt or mail command. + Please make sure you can send a test mail with at least one of the following + commands run by your sync user: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +$ echo "your test message" | mutt -x -s "This is a test message" your@mail.tld +\end_layout + +\begin_layout Plain Layout + +$ echo "your test message" | mail -s "This is a test message" your@mail.tld +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Check your antispam if you don't get your message. + If you still don't get your message, check your distributions documentation + about the mail command. +\end_layout + +\begin_layout Standard +If you run on windows environment, please make sure you can launch sendemail.exe + (found +\begin_inset CommandInset href +LatexCommand href +name "here" +target "http://caspian.dotconf.net/menu/Software/SendEmail/" + +\end_inset + +). +\end_layout + +\begin_layout Subsection +\begin_inset CommandInset label +LatexCommand label +name "sub:Enhancing-remote-backup" + +\end_inset + +Enhancing remote system security +\end_layout + +\begin_layout Standard +We may want to secure a password-less ssh access by removing non necessary + services offered by SSH. + Edit the file ~/.ssh/authorized_keys created earlier and add the following + line in the beginning of the file: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Also, we may want to prevent any host except of our remote system to passwordles +s connect. + Add the following line: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +from=*.my.remote.servers.domain.tld +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Your authorized_keys file should look like this: +\end_layout + +\begin_layout Standard +\begin_inset listings +lstparams "breaklines=true" +inline false +status open + +\begin_layout Plain Layout + +from="*.mydomain.tld",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no- +pty ssh-rsa yourkey== syncuser@host.tld +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +More security enhancing +\end_layout + +\begin_layout Standard +We may also restrict the ssh session to only a couple of commands we'll + need. + Osync comes with a script called +\emph on +ssh_filter.sh +\emph default + that will only execute commands Osync might send. + Once again edit your authorized_keys file and add the following. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +command="/usr/local/bin/ssh_filter.sh" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Your file should then look like this: +\end_layout + +\begin_layout Standard +\begin_inset listings +lstparams "breaklines=true,showstringspaces=false" +inline false +status open + +\begin_layout Plain Layout + +from="*.mydomain.tld",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no- +pty,command="/usr/local/bin/ssh_filter.sh" ssh-rsa yourkey== syncuser@remotesyste +m.tld +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Copy then the script ssh_filter.sh to /usr/local/bin on the remote system. + Don't forget to make it executable and make it owned by root +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# chmod 755 /usr/local/bin/ssh_filter.sh +\end_layout + +\begin_layout Plain Layout + +# chown root:root /usr/local/bin/ssh_filter.sh +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Now, only the commands +\begin_inset Quotes eld +\end_inset + +find, du, rsync, echo and sudo +\begin_inset Quotes erd +\end_inset + + may be executed via the ssh tunnel. + You may enable / disable the usage of sudo command by editing the following + value in the ssh_filter.sh script: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SUDO_EXEC=yes +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Also, adding remote pre- and postexecution commands in your configuration + files will not work if you use the ssh filter. + You'll have to add your main command in ssh_filter.sh. + Example if you want to perform remote snapshots you'll have to allow one + of the following: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +CMD1=zfs +\end_layout + +\begin_layout Plain Layout + +CMD2=xfs +\end_layout + +\begin_layout Plain Layout + +CMD3=lvm +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +\begin_inset CommandInset label +LatexCommand label +name "sub:More-security-(or" + +\end_inset + +Security for the paranoid +\end_layout + +\begin_layout Standard +Executing rsync as superuser is a security risk. + A way to prevent rsync usage allowing only a symlink to be executed. + Thus, a attacker script using rsync would not work. + This kind of security is called +\begin_inset Quotes eld +\end_inset + +security by obscurity +\begin_inset Quotes erd +\end_inset + + and should generally not be the only security process, but makes any attack + harder. + First, let's create a symlink to rsync called let's say o_rsync +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# ln -s $(which rsync) $(dirname $(which rsync))/o_rsync +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Now edit ssh_filter.sh and change the following value: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RSYNC_EXECUTABLE=o_rsync +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Also, edit RSYNC_EXECUTABLE value on any of your sync configuration files + and you're done. + +\end_layout + +\begin_layout Section +Running Osync in quicksync mode +\end_layout + +\begin_layout Standard +You must first download a copy of osync to your computer, and make it executable. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# chmod +x osync.sh +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +You might consider copying osync to +\emph on +/usr/local/bin +\end_layout + +\begin_layout Standard +Then you might just launch it to sync two local dirs like this: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# ./osync.sh --master=/path/to/dir1 --slave=/path/to/dir2 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +You also may want to sync a remote directory. + +\end_layout + +\begin_layout Standard +You may specify an alternate SSH port directly in the URI. + When ommited, SSH port 22 is used. +\end_layout + +\begin_layout Standard +Also, if not set, the default RSA key will be read from ~/.ssh/id_rsa +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# ./osync.sh --master=/path/to/dir1 --slave=ssh://remoteuser@remotehost.com//path/t +o/dir2 +\end_layout + +\begin_layout Plain Layout + +# ./osync.sh --master=/path/to/dir2 --slave=ssh://remoteuser@remotehost.com:22//pat +h/to/dir2 --rsakey=/home/user/.ssh/other_key +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Section +Running Osync with a full blown configuration file +\end_layout + +\begin_layout Standard +Running Osync with a configuration will do just the same as in quicksync + mode, except the fact that you have much more control of what's going on. +\end_layout + +\begin_layout Standard +A sample configuration file is called sync.conf and is included with osync. + You may edit this file to fit your needs. +\end_layout + +\begin_layout Standard +Every option of the configuration file is explained in the appendix. +\end_layout + +\begin_layout Standard +Once you've setup a file according to your needs, you may go for a test + run. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# ./osync.sh /path/to/my_sync.conf --dry --verbose +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Osync should now enumerate which changes will be done on both sides. +\end_layout + +\begin_layout Standard +If everything worked out right, you might process the actual sync process. + +\end_layout + +\begin_layout Standard +Remember that a full configuration file specifies a maximum execution delay. + Initial sync tasks can take a huge amount of time depending on bandwidth + between replicas, in that case you might add parameter --no-maxtime to + your first sync run so execution time won't be enforced. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# ./osync.sh /path/to/my_sync.conf --no-maxtime +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Creating a regular sync scenario is quite simple as long as you don't schedule + two times the same sync task in a shorter time span than your HARD_MAX_EXEC_TIM +E_TOTAL value. + Just create a crontab entry and add parameter --silent so your local mailbox + won't get filled up. + Example, having async scheduled every hour in /etc/crontab +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +00 * * * * localsyncuser /usr/local/bin/osync.sh /home/localsyncuser/your_sync.con +f --silent +\end_layout + +\end_inset + +You may find the sync log under /var/log/osync-version-your_sync.log or under + the current directory if /var/log is not writable. +\end_layout + +\begin_layout Section +Running Osync on file changes +\end_layout + +\begin_layout Subsection +By hand +\end_layout + +\begin_layout Standard +Osync may also run in file monitor mode. + In this mode, osync monitors the master replica, and runs a synchronization + everytime the is file activity on master replica. + With this mode, you do not need cron anymore. + Be aware that only master replica is monitored, and slave replica sync + updates only occur when master replica modifications happen. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# ./osync.sh /path/to/my_sync.conf --on-changes +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Subsection +As a system service +\end_layout + +\begin_layout Standard +If you plan to run Osync on a regular basis in file monitor mode, you might + consider installing it as a system service. +\end_layout + +\begin_layout Standard +From the directory you downloaded Osync, run the following to make Osync + a service: +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +# cp osync.sh /usr/local/bin +\end_layout + +\begin_layout Plain Layout + +# cp osync-srv /etc/init.d +\end_layout + +\begin_layout Plain Layout + +# mkdir /etc/osync +\end_layout + +\begin_layout Plain Layout + +# cp sync.conf /etc/osync +\end_layout + +\begin_layout Plain Layout + +# service osync-srv start +\end_layout + +\begin_layout Plain Layout + +# chkconfig osync-srv on +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Osync then scans for *.conf files in /etc/osync and will run an instance + per configuration file. +\end_layout + +\begin_layout Standard +Service control just works like with standard system services. +\end_layout + +\begin_layout Section +Configuration appendix +\end_layout + +\begin_layout Subsection +Quicksync only command line parameters +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--master= +\begin_inset Quotes srd +\end_inset + + +\begin_inset Quotes srd +\end_inset + + Master replica path. + Will contain state and backup directory (is mandatory) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--slave Local or remote slave replica path. + Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is + mandatory) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--rsakey Alternative path to rsa private key for ssh connection to slave + replica +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--sync-id Optional sync task name to identify this synchronization task + when using multiple slaves +\end_layout + +\begin_layout Subsection +Universal command line parameters +\end_layout + +\begin_layout Standard +When run without any parameter, Osync will show usage. +\end_layout + +\begin_layout Standard +Both quicksync and config file modes can take the following optional parameters: +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--dry Will make osync run a simulation only +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--silent Will run osync silently, to be used in a cron schedule +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--verbose Will run osync with detailed output, including changed and deleted + files lists on both sides +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--stats Will add rsync transfer statistics to verbose output +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--no-maxtime Will disable MAX_EXEC_TIME checks, so a task can take as long + as it needs. + This is usefull for performing initial big sync operations +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--force-unlock Will override any existing active or dead locks on master + and slave replica +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--on-changes Will launch a sync task after a short wait period if there + is some file activity on master replica. + You should try daemon mode instead +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +--help Will print Osync version and usage +\end_layout + +\begin_layout Subsection +Full list of configuration file parameters +\end_layout + +\begin_layout Standard +Set this to whatever you want to identify your sync task. + This value is also in the log name and in the warning mails. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SYNC_ID=name_of_your_sync +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Master directory to sync (master replica), must be on the system you're + running osync on. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +MASTER_SYNC_DIR="/some/path" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Slave directory to sync (slave replica), can be on the same system you're + running osync on or another remote system, reachable via an SSH tunnel. +\end_layout + +\begin_layout Standard +Slave directory can be a SSH uri like +\begin_inset Quotes sld +\end_inset + +ssh://user@host.com:1234//some/other/path +\begin_inset Quotes srd +\end_inset + + +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SLAVE_SYNC_DIR="/some/other/path" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Location of the private RSA key. + If left empty, the default path +\begin_inset Quotes sld +\end_inset + +~/.ssh/id_rsa +\begin_inset Quotes srd +\end_inset + + will be used. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Tells osync to create master or slave directories if they don't exist. + Default is no. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +CREATE_DIRS=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +By default, leaving this empty sets the log file to /var/log/osync_SYNC_ID.log. + You might change this to specify a personalized log file. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +LOGFILE="" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +List of directories to exclude in sync task on both sides (rsync patterns, + wildcards will work). + Paths are relative to both replicas paths. + List is separated by PATH_SEPARATOR_CHAR defined below. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RSYNC_EXCLUDE_PATTERN="tmp;archives;whatever path" +\end_layout + +\end_inset + +File that contains the list of directories or files to exclude from sync + on both sides. + Leave this empty if you don't want to use an exclusion file. + This file has to be in the same directory as the config file. + Paths are relative to sync dirs. + One element per line. + +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RSYNC_EXCLUDE_FROM="exclude.list" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Path separator char for RSYNC_EXCLUDE_PATTERN, you might change this in + the unholy case that your filenames contains semicolons. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +PATH_SEPARATOR_CHAR=";" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Generate an alert if master or slave replicas have less space than the following + given value in kilobytes. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +MINIMUM_SPACE=10240 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Bandwidth limit in kilobytes / second. + Leave this to zero to disable limitation. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +BANDWIDTH=0 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Synchronization tasks may be executed as root if you enable the following + parameter. + See prerequisites in chapter +\begin_inset CommandInset ref +LatexCommand ref +reference "sub:Performing-superuser-backups" + +\end_inset + +. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SUDO_EXEC=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Paranoia option. + Don't change this unless you read chapter +\begin_inset CommandInset ref +LatexCommand ref +reference "sub:More-security-(or" + +\end_inset + +. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RSYNC_EXECUTABLE=rsync +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Enable / disable ssh compression. + Leave this enabled unless your connection to remote system is high speed + (LAN) +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SSH_COMPRESSION=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Ping remote host before launching synchronization. + Be sure the host responds to ping. + Failing to ping will skip current task. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +REMOTE_HOST_PING=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +Check for internet access by pinging one or more hosts before launching + remote sync task. + Leave this empty do disable the check. + Failing to ping will stop execution. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +REMOTE_3RD_PARTY_HOST="www.kernel.org" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +Misc settings +\end_layout + +\begin_layout Standard +Preserve ACLs. + Please check that your filesystem supports ACLs and is mounted with it's + support or rsync will get you loads of errors. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +PRESERVE_ACL=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Preserve Xattr. + The same applies as for ACLs +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +PRESERVE_XATTR=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Transforms symlinks into referent files/dirs when syncing replicas. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +COPY_SYMLINKS=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Treat symlinked dirs as dirs. + CAUTION: This also follows symlinks outside of the replica root. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +KEEP_DIRLINKS=no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Preserve hard links. + Make sure source and target FS can manage hard links or you will lose them. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +PRESERVE_HARDLINKS=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Use rsync compression for file transfers. + Leave this disabled unless your're not using SSH compression. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RSYNC_COMPRESS=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Maximum execution time (in seconds) for sync process. + Soft value generates a warning only. + Hard value generates a warning and stops sync task. +\end_layout + +\begin_layout Standard +You may set this to 0 to disable time checks. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SOFT_MAX_EXEC_TIME_FILE_TASK=7200 +\end_layout + +\begin_layout Plain Layout + +HARD_MAX_EXEC_TIME_FILE_TASK=10600 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Minimum time (in seconds) in file monitor /daemon mode between modification + detection and sync task in order to let copy operations finish. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +MIN_WAIT=60 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +Conflict and deletion option +\end_layout + +\begin_layout Standard +Enabling this option will keep a backup of a file on the target replica + if it gets updated from the source replica. + Backups will be made to .osync_workdir/backups +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +CONFLICT_BACKUP=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Keep multiple backup versions of the same file. + Warning, This can be very space consuming. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +CONFLICT_BACKUP_MULTIPLE=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Osync will clean backup files after a given number of days. + Setting this to 0 will disable cleaning and keep backups forever. + Warning: This can be very space consuming. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +CONFLICT_BACKUP_DAYS=30 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +If the same file exists on both replicas, newer version will be synced. + However, if both files have the same timestamp but differ, CONFILCT_PREVALANCE + sets winner replica. +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +CONFLICT_PREVALANCE=master|slave +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +On deletition propagation to the target replica, a backup of the deleted + files can be kept. + Deletions will be kept in .osync_workdir/deleted +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SOFT_DELETE=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Osync will clean deleted files after a given number of days. + Setting this to 0 will disable cleaning and keep deleted files forever. + Warning: This can be very space consuming. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SOFT_DELETE_DAYS=30 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +Resuming options +\end_layout + +\begin_layout Standard +Try to resume an aborted sync task +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RESUME_SYNC=yes|no +\end_layout + +\end_inset + + Number maximum resume tries before initating a fresh sync. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +RESUME_TRY=2 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +When a pidlock exists on slave replica that does not correspond to master's + sync-id, force pidlock removal. + Be carefull with this option if you have multiple masters. +\end_layout + +\begin_layout Standard +\begin_inset listings +lstparams "breaklines=true" +inline false +status open + +\begin_layout Plain Layout + +FORCE_STRANGER_LOCK_RESUME=no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Keep partial uploads that can be resumed on next run, experimantal feature. + This can be very usefull if big files must get updated though slow links. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +PARTIAL=no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +Alert Options +\end_layout + +\begin_layout Standard +List of alert mails separated by spaces +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +DESTINATION_MAILS="your@alert.tld" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Windows (MSYS environment) only mail options (used with sendemail.exe from + Brandon Zehm) +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +SENDER_MAIL="alert@your.system.tld" +\end_layout + +\begin_layout Plain Layout + +SMTP_SERVER=smtp.your.isp.tld +\end_layout + +\begin_layout Plain Layout + +SMTP_USER=optional_smtp_user +\end_layout + +\begin_layout Plain Layout + +SMTP_PASSWORD=optional_smtp_password +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +Execution hooks +\end_layout + +\begin_layout Standard +Commands can will be run before and / or after sync process (remote execution + will only happen if REMOTE_SYNC is set). + Multiple commands can be semicolon separated. +\end_layout + +\begin_layout Standard +Command(s) to run locally before sync process starts. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +LOCAL_RUN_BEFORE_CMD="" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Command(s) to run locally if sync process finishes. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +LOCAL_RUN_AFTER_CMD="" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Command(s) to run on remote system before sync process starts. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +REMOTE_RUN_BEFORE_CMD="" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Command(s) to run on remote system if sync process finishes. +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +REMOTE_RUN_AFTER_CMD="" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Max execution time of commands before they get force killed. + Leave 0 if you don't wan't this to happen. + Time is specified in seconds. + MAX_EXEC_TIME_PER_CMD_BEFORE=0 +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +MAX_EXEC_TIME_PER_CMD_AFTER=0 +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Standard +Stops Osync execution if one of the above commands fail +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +STOP_ON_CMD_ERROR=yes|no +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Section +Troubleshooting +\end_layout + +\begin_layout Standard +Osync has been tested successfully on multiple systems for a wide variety + of sync plans. + Please check the following steps before requesting help. +\end_layout + +\begin_layout Subsection +Local-local sync +\end_layout + +\begin_layout Standard +Osync logs every of it's actions to /var/log/osync-version-your_sync_id.log + (or current directory if /var/log is not writable). +\end_layout + +\begin_layout Standard +Please check the log file if something went wrong. +\end_layout + +\begin_layout Standard +You might try running osync as root to check if your problem is filesystem + permission related. +\end_layout + +\begin_layout Standard +You might add --verbose option to see what actually happens. +\end_layout + +\begin_layout Subsection +Local-remote sync +\end_layout + +\begin_layout Standard +Remote synchronization is a bit more tricky. +\end_layout + +\begin_layout Standard +You might check that you can log in remotely with the command +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +$ ssh -p 22 remotesyncuser@remotehost.tld +\end_layout + +\end_inset + +Also, you might check that you can use rsync command remotely +\end_layout + +\begin_layout Standard +\begin_inset listings +inline false +status open + +\begin_layout Plain Layout + +$ ssh -p 22 remotesyncuser@remotehost.tld rsync --help +\end_layout + +\end_inset + +You can temporarily disable ssh security by removing lines you added in + chapter +\begin_inset CommandInset ref +LatexCommand ref +reference "sub:Enhancing-remote-backup" + +\end_inset + +. + Additionnaly, you can check ssh_filter log in ~/.ssh/ssh_filter.log on the + remote system. + You might try running osync with SUDO_EXEC to check if your problem is + user permission related. +\end_layout + +\begin_layout Standard +\begin_inset CommandInset line +LatexCommand rule +offset "0.5ex" +width "100col%" +height "1pt" + +\end_inset + + +\end_layout + +\end_body +\end_document