From 497da8df0469c6669660a9fd6f5a34bf24412999 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 5 Feb 2017 15:53:32 +0100 Subject: [PATCH 01/10] docs: init: fix markup/typos --- src/borg/archiver.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index b5fb165c1..0746cbdcb 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1765,32 +1765,33 @@ class Archiver: Encryption modes ++++++++++++++++ - repokey and keyfile use AES-CTR-256 for encryption and HMAC-SHA256 for + `repokey` and `keyfile` use AES-CTR-256 for encryption and HMAC-SHA256 for authentication in an encrypt-then-MAC (EtM) construction. The chunk ID hash is HMAC-SHA256 as well (with a separate key). These modes are compatible with borg 1.0.x. - repokey-blake2 and keyfile-blake2 are also authenticated encryption modes, + `repokey-blake2` and `keyfile-blake2` are also authenticated encryption modes, but use BLAKE2b-256 instead of HMAC-SHA256 for authentication. The chunk ID hash is a keyed BLAKE2b-256 hash. - These modes are new and not compatible with borg 1.0.x. + These modes are new and *not* compatible with borg 1.0.x. - "authenticated" mode uses no encryption, but authenticates repository contents + `authenticated` mode uses no encryption, but authenticates repository contents through the same keyed BLAKE2b-256 hash as the other blake2 modes (it uses it as chunk ID hash). The key is stored like repokey. This mode is new and not compatible with borg 1.0.x. - "none" mode uses no encryption and no authentication. It uses sha256 as chunk + `none` mode uses no encryption and no authentication. It uses sha256 as chunk ID hash. Not recommended, rather consider using an authenticated or authenticated/encrypted mode. This mode is compatible with borg 1.0.x. Hardware acceleration will be used automatically. - On modern Intel/AMD CPUs (except very cheap ones), AES is usually hw - accelerated. BLAKE2b is faster than sha256 on Intel/AMD 64bit CPUs. + On modern Intel/AMD CPUs (except very cheap ones), AES is usually + hardware-accelerated. BLAKE2b is faster than SHA256 on Intel/AMD 64bit CPUs, + which makes `authenticated` faster than `none`. - On modern ARM CPUs, NEON provides hw acceleration for sha256 making it faster + On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster than BLAKE2b-256 there. """) subparser = subparsers.add_parser('init', parents=[common_parser], add_help=False, @@ -1804,7 +1805,7 @@ class Archiver: subparser.add_argument('-e', '--encryption', dest='encryption', choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'), default=None, - help='select encryption key mode (default: "%(default)s")') + help='select encryption key mode') subparser.add_argument('-a', '--append-only', dest='append_only', action='store_true', help='create an append-only mode repository') From 0710bbd40e07f9423e2977778fd67e3159a0e465 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 5 Feb 2017 21:17:21 +0100 Subject: [PATCH 02/10] docs: create: move item flags to main doc --- docs/man/borg-create.1 | 57 +++++++++++++++- docs/man/borg-list.1 | 151 +++++++++++++++++++++-------------------- docs/usage.rst | 40 ----------- setup.py | 11 ++- src/borg/archiver.py | 117 ++++++++++++++++++++++--------- 5 files changed, 228 insertions(+), 148 deletions(-) diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index f3e3c9be0..3e30a343e 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-02-12" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -224,6 +224,61 @@ exclude foo/.bundler/gems. In borg it will not, you need to use \-\-exclude \(aq*/.bundler/gems\(aq to get the same effect. See \fBborg help patterns\fP for more information. .UNINDENT +.SH NOTES +.SS Item flags +.sp +\fB\-\-list\fP outputs a list of all files, directories and other +file system items it considered (no matter whether they had content changes +or not). For each item, it prefixes a single\-letter flag that indicates type +and/or status of the item. +.sp +If you are interested only in a subset of that output, you can give e.g. +\fB\-\-filter=AME\fP and it will only show regular files with A, M or E status (see +below). +.sp +A uppercase character represents the status of a regular file relative to the +"files" cache (not relative to the repo \-\- this is an issue if the files cache +is not used). Metadata is stored in any case and for \(aqA\(aq and \(aqM\(aq also new data +chunks are stored. For \(aqU\(aq all data chunks refer to already existing chunks. +.INDENT 0.0 +.IP \(bu 2 +\(aqA\(aq = regular file, added (see also \fIa_status_oddity\fP in the FAQ) +.IP \(bu 2 +\(aqM\(aq = regular file, modified +.IP \(bu 2 +\(aqU\(aq = regular file, unchanged +.IP \(bu 2 +\(aqE\(aq = regular file, an error happened while accessing/reading \fIthis\fP file +.UNINDENT +.sp +A lowercase character means a file type other than a regular file, +borg usually just stores their metadata: +.INDENT 0.0 +.IP \(bu 2 +\(aqd\(aq = directory +.IP \(bu 2 +\(aqb\(aq = block device +.IP \(bu 2 +\(aqc\(aq = char device +.IP \(bu 2 +\(aqh\(aq = regular file, hardlink (to already seen inodes) +.IP \(bu 2 +\(aqs\(aq = symlink +.IP \(bu 2 +\(aqf\(aq = fifo +.UNINDENT +.sp +Other flags used include: +.INDENT 0.0 +.IP \(bu 2 +\(aqi\(aq = backup data was read from standard input (stdin) +.IP \(bu 2 +\(aq\-\(aq = dry run, item was \fInot\fP backed up +.IP \(bu 2 +\(aqx\(aq = excluded, item was \fInot\fP backed up +.IP \(bu 2 +\(aq?\(aq = missing status code (if you see this, please file a bug report!) +.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-check(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index d6e5a4ec8..74e5a2780 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-02-05" "" "borg backup tool" +.TH BORG-LIST 1 "2017-02-12" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . @@ -38,6 +38,81 @@ borg list REPOSITORY_OR_ARCHIVE PATH This command lists the contents of a repository or an archive. .sp See the "borg help patterns" command for more help on exclude patterns. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.INDENT 0.0 +.TP +.B REPOSITORY_OR_ARCHIVE +repository/archive to list contents of +.TP +.B PATH +paths to list; patterns are supported +.UNINDENT +.SS optional arguments +.INDENT 0.0 +.TP +.B \-\-short +only print file/directory names, nothing else +.TP +.B \-\-format\fP,\fB \-\-list\-format +specify format for file listing +(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") +.TP +.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN +exclude paths matching PATTERN +.TP +.BI \-\-exclude\-from \ EXCLUDEFILE +read exclude patterns from EXCLUDEFILE, one per line +.UNINDENT +.SS filters +.INDENT 0.0 +.TP +.B \-P\fP,\fB \-\-prefix +only consider archive names starting with this prefix +.TP +.B \-\-sort\-by +Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp +.TP +.BI \-\-first \ N +consider first N archives after other filters were applied +.TP +.BI \-\-last \ N +consider last N archives after other filters were applied +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg list /path/to/repo +Monday Mon, 2016\-02\-15 19:15:11 +repo Mon, 2016\-02\-15 19:26:54 +root\-2016\-02\-15 Mon, 2016\-02\-15 19:36:29 +newname Mon, 2016\-02\-15 19:50:19 +\&... + +$ borg list /path/to/repo::root\-2016\-02\-15 +drwxr\-xr\-x root root 0 Mon, 2016\-02\-15 17:44:27 . +drwxrwxr\-x root root 0 Mon, 2016\-02\-15 19:04:49 bin +\-rwxr\-xr\-x root root 1029624 Thu, 2014\-11\-13 00:08:51 bin/bash +lrwxrwxrwx root root 0 Fri, 2015\-03\-27 20:24:26 bin/bzcmp \-> bzdiff +\-rwxr\-xr\-x root root 2140 Fri, 2015\-03\-27 20:24:22 bin/bzdiff +\&... + +$ borg list /path/to/repo::archiveA \-\-list\-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" +drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 . +drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code +drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject +\-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.SH NOTES .sp The following keys are available for \-\-format: .INDENT 0.0 @@ -162,80 +237,6 @@ health: either "healthy" (file ok) or "broken" (if file has all\-zero replacemen .UNINDENT .UNINDENT .UNINDENT -.SH OPTIONS -.sp -See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY_OR_ARCHIVE -repository/archive to list contents of -.TP -.B PATH -paths to list; patterns are supported -.UNINDENT -.SS optional arguments -.INDENT 0.0 -.TP -.B \-\-short -only print file/directory names, nothing else -.TP -.B \-\-format\fP,\fB \-\-list\-format -specify format for file listing -(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") -.TP -.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN -exclude paths matching PATTERN -.TP -.BI \-\-exclude\-from \ EXCLUDEFILE -read exclude patterns from EXCLUDEFILE, one per line -.UNINDENT -.SS filters -.INDENT 0.0 -.TP -.B \-P\fP,\fB \-\-prefix -only consider archive names starting with this prefix -.TP -.B \-\-sort\-by -Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp -.TP -.BI \-\-first \ N -consider first N archives after other filters were applied -.TP -.BI \-\-last \ N -consider last N archives after other filters were applied -.UNINDENT -.SH EXAMPLES -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ borg list /path/to/repo -Monday Mon, 2016\-02\-15 19:15:11 -repo Mon, 2016\-02\-15 19:26:54 -root\-2016\-02\-15 Mon, 2016\-02\-15 19:36:29 -newname Mon, 2016\-02\-15 19:50:19 -\&... - -$ borg list /path/to/repo::root\-2016\-02\-15 -drwxr\-xr\-x root root 0 Mon, 2016\-02\-15 17:44:27 . -drwxrwxr\-x root root 0 Mon, 2016\-02\-15 19:04:49 bin -\-rwxr\-xr\-x root root 1029624 Thu, 2014\-11\-13 00:08:51 bin/bash -lrwxrwxrwx root root 0 Fri, 2015\-03\-27 20:24:26 bin/bzcmp \-> bzdiff -\-rwxr\-xr\-x root root 2140 Fri, 2015\-03\-27 20:24:22 bin/bzdiff -\&... - -$ borg list /path/to/repo::archiveA \-\-list\-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" -drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 . -drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code -drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject -\-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext -\&... -.ft P -.fi -.UNINDENT -.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP, \fIborg\-info(1)\fP, \fIborg\-diff(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-patterns(1)\fP diff --git a/docs/usage.rst b/docs/usage.rst index d160bc0a8..bdc6b23df 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -851,46 +851,6 @@ Additional Notes Here are misc. notes about topics that are maybe not covered in enough detail in the usage section. -Item flags -~~~~~~~~~~ - -``borg create --list`` outputs a list of all files, directories and other -file system items it considered (no matter whether they had content changes -or not). For each item, it prefixes a single-letter flag that indicates type -and/or status of the item. - -If you are interested only in a subset of that output, you can give e.g. -``--filter=AME`` and it will only show regular files with A, M or E status (see -below). - -A uppercase character represents the status of a regular file relative to the -"files" cache (not relative to the repo -- this is an issue if the files cache -is not used). Metadata is stored in any case and for 'A' and 'M' also new data -chunks are stored. For 'U' all data chunks refer to already existing chunks. - -- 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ) -- 'M' = regular file, modified -- 'U' = regular file, unchanged -- 'E' = regular file, an error happened while accessing/reading *this* file - -A lowercase character means a file type other than a regular file, -borg usually just stores their metadata: - -- 'd' = directory -- 'b' = block device -- 'c' = char device -- 'h' = regular file, hardlink (to already seen inodes) -- 's' = symlink -- 'f' = fifo - -Other flags used include: - -- 'i' = backup data was read from standard input (stdin) -- '-' = dry run, item was *not* backed up -- 'x' = excluded, item was *not* backed up -- '?' = missing status code (if you see this, please file a bug report!) - - --chunker-params ~~~~~~~~~~~~~~~~ The chunker params influence how input files are cut into pieces (chunks) diff --git a/setup.py b/setup.py index 2062221ec..6e57a5eda 100644 --- a/setup.py +++ b/setup.py @@ -214,6 +214,8 @@ class build_usage(Command): def run(self): print('generating usage docs') + import borg + borg.doc_mode = 'build_man' if not os.path.exists('docs/usage'): os.mkdir('docs/usage') # allows us to build docs without the C modules fully loaded during help generation @@ -361,6 +363,8 @@ class build_man(Command): def run(self): print('building man pages (in docs/man)', file=sys.stderr) + import borg + borg.doc_mode = 'build_man' os.makedirs('docs/man', exist_ok=True) # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver @@ -399,7 +403,8 @@ class build_man(Command): write('\n') self.write_heading(write, 'DESCRIPTION') - write(parser.epilog) + description, _, notes = parser.epilog.partition('\n.. man NOTES') + write(description) self.write_heading(write, 'OPTIONS') write('See `borg-common(1)` for common options of Borg commands.') @@ -408,6 +413,10 @@ class build_man(Command): self.write_examples(write, command) + if notes: + self.write_heading(write, 'NOTES') + write(notes) + self.write_see_also(write, man_title) self.gen_man_page(man_title, doc.getvalue()) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0746cbdcb..65d39bd90 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -25,6 +25,7 @@ logger = create_logger() import msgpack +import borg from . import __version__ from . import helpers from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special @@ -1657,6 +1658,16 @@ class Archiver: return args def build_parser(self, prog=None): + def process_epilog(epilog): + epilog = textwrap.dedent(epilog).splitlines() + try: + mode = borg.doc_mode + except AttributeError: + mode = 'command-line' + if mode in ('command-line', 'build_usage'): + epilog = [line for line in epilog if not line.startswith('.. man')] + return '\n'.join(epilog) + common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_group = common_parser.add_argument_group('Common options') @@ -1703,7 +1714,7 @@ class Archiver: help='show version number and exit') subparsers = parser.add_subparsers(title='required arguments', metavar='') - serve_epilog = textwrap.dedent(""" + serve_epilog = process_epilog(""" This command starts a repository server process. This command is usually not used manually. """) subparser = subparsers.add_parser('serve', parents=[common_parser], add_help=False, @@ -1718,7 +1729,7 @@ class Archiver: subparser.add_argument('--append-only', dest='append_only', action='store_true', help='only allow appending to repository segment files') - init_epilog = textwrap.dedent(""" + init_epilog = process_epilog(""" This command initializes an empty repository. A repository is a filesystem directory containing the deduplicated data from zero or more archives. @@ -1809,7 +1820,7 @@ class Archiver: subparser.add_argument('-a', '--append-only', dest='append_only', action='store_true', help='create an append-only mode repository') - check_epilog = textwrap.dedent(""" + check_epilog = process_epilog(""" The check command verifies the consistency of a repository and the corresponding archives. First, the underlying repository data files are checked: @@ -1895,7 +1906,7 @@ class Archiver: key_parsers = subparser.add_subparsers(title='required arguments', metavar='') subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser)) - key_export_epilog = textwrap.dedent(""" + key_export_epilog = process_epilog(""" If repository encryption is used, the repository is inaccessible without the key. This command allows to backup this essential key. @@ -1928,7 +1939,7 @@ class Archiver: default=False, help='Create an export suitable for printing and later type-in') - key_import_epilog = textwrap.dedent(""" + key_import_epilog = process_epilog(""" This command allows to restore a key previously backed up with the export command. @@ -1950,7 +1961,7 @@ class Archiver: default=False, help='interactively import from a backup done with --paper') - change_passphrase_epilog = textwrap.dedent(""" + change_passphrase_epilog = process_epilog(""" The key files used for repository encryption are optionally passphrase protected. This command can be used to change this passphrase. """) @@ -1973,7 +1984,7 @@ class Archiver: subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) - migrate_to_repokey_epilog = textwrap.dedent(""" + migrate_to_repokey_epilog = process_epilog(""" This command migrates a repository from passphrase mode (removed in Borg 1.0) to repokey mode. @@ -2000,7 +2011,7 @@ class Archiver: subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) - create_epilog = textwrap.dedent(""" + create_epilog = process_epilog(""" This command creates a backup archive containing all files found while recursively traversing all paths specified. When giving '-' as path, borg will read data from standard input and create a file 'stdin' in the created archive from that @@ -2023,6 +2034,47 @@ class Archiver: See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. + + .. man NOTES + + Item flags + ++++++++++ + + ``--list`` outputs a list of all files, directories and other + file system items it considered (no matter whether they had content changes + or not). For each item, it prefixes a single-letter flag that indicates type + and/or status of the item. + + If you are interested only in a subset of that output, you can give e.g. + ``--filter=AME`` and it will only show regular files with A, M or E status (see + below). + + A uppercase character represents the status of a regular file relative to the + "files" cache (not relative to the repo -- this is an issue if the files cache + is not used). Metadata is stored in any case and for 'A' and 'M' also new data + chunks are stored. For 'U' all data chunks refer to already existing chunks. + + - 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ) + - 'M' = regular file, modified + - 'U' = regular file, unchanged + - 'E' = regular file, an error happened while accessing/reading *this* file + + A lowercase character means a file type other than a regular file, + borg usually just stores their metadata: + + - 'd' = directory + - 'b' = block device + - 'c' = char device + - 'h' = regular file, hardlink (to already seen inodes) + - 's' = symlink + - 'f' = fifo + + Other flags used include: + + - 'i' = backup data was read from standard input (stdin) + - '-' = dry run, item was *not* backed up + - 'x' = excluded, item was *not* backed up + - '?' = missing status code (if you see this, please file a bug report!) """) subparser = subparsers.add_parser('create', parents=[common_parser], add_help=False, @@ -2123,7 +2175,7 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, help='paths to archive') - extract_epilog = textwrap.dedent(""" + extract_epilog = process_epilog(""" This command extracts the contents of an archive. By default the entire archive is extracted but a subset of files and directories can be selected by passing a list of ``PATHs`` as arguments. The file selection can further @@ -2174,7 +2226,7 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to extract; patterns are supported') - diff_epilog = textwrap.dedent(""" + diff_epilog = process_epilog(""" This command finds differences (file contents, user/group/mode) between archives. A repository location and an archive name must be specified for REPO_ARCHIVE1. @@ -2222,7 +2274,7 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths of items inside the archives to compare; patterns are supported') - rename_epilog = textwrap.dedent(""" + rename_epilog = process_epilog(""" This command renames an archive in the repository. This results in a different archive ID. @@ -2240,7 +2292,7 @@ class Archiver: type=archivename_validator(), help='the new archive name to use') - delete_epilog = textwrap.dedent(""" + delete_epilog = process_epilog(""" This command deletes an archive from the repository or the complete repository. Disk space is reclaimed accordingly. If you delete the complete repository, the local cache for it (if any) is also deleted. @@ -2271,13 +2323,16 @@ class Archiver: help='archive or repository to delete') self.add_archives_filters_args(subparser) - list_epilog = textwrap.dedent(""" + list_epilog = process_epilog(""" This command lists the contents of a repository or an archive. See the "borg help patterns" command for more help on exclude patterns. + .. man NOTES + The following keys are available for --format: + """) + BaseFormatter.keys_help() + textwrap.dedent(""" Keys for listing repository archives: @@ -2312,7 +2367,7 @@ class Archiver: help='paths to list; patterns are supported') self.add_archives_filters_args(subparser) - mount_epilog = textwrap.dedent(""" + mount_epilog = process_epilog(""" This command mounts an archive as a FUSE filesystem. This can be useful for browsing an archive or restoring individual files. Unless the ``--foreground`` option is given the command will run in the background until the filesystem @@ -2363,7 +2418,7 @@ class Archiver: help='Extra mount options') self.add_archives_filters_args(subparser) - umount_epilog = textwrap.dedent(""" + umount_epilog = process_epilog(""" This command un-mounts a FUSE filesystem that was mounted with ``borg mount``. This is a convenience wrapper that just calls the platform-specific shell @@ -2378,7 +2433,7 @@ class Archiver: subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, help='mountpoint of the filesystem to umount') - info_epilog = textwrap.dedent(""" + info_epilog = process_epilog(""" This command displays detailed information about the specified archive or repository. Please note that the deduplicated sizes of the individual archives do not add @@ -2401,7 +2456,7 @@ class Archiver: help='archive or repository to display information about') self.add_archives_filters_args(subparser) - break_lock_epilog = textwrap.dedent(""" + break_lock_epilog = process_epilog(""" This command breaks the repository and cache locks. Please use carefully and only while no borg process (on any machine) is trying to access the Cache or the Repository. @@ -2416,7 +2471,7 @@ class Archiver: type=location_validator(archive=False), help='repository for which to break the locks') - prune_epilog = textwrap.dedent(""" + prune_epilog = process_epilog(""" The prune command prunes a repository by deleting all archives not matching any of the specified retention options. This command is normally used by automated backup scripts wanting to keep a certain number of historic backups. @@ -2502,7 +2557,7 @@ class Archiver: type=location_validator(archive=False), help='repository to prune') - upgrade_epilog = textwrap.dedent(""" + upgrade_epilog = process_epilog(""" Upgrade an existing Borg repository. Borg 1.x.y upgrades @@ -2592,7 +2647,7 @@ class Archiver: type=location_validator(archive=False), help='path to the repository to be upgraded') - recreate_epilog = textwrap.dedent(""" + recreate_epilog = process_epilog(""" Recreate the contents of existing archives. This is an *experimental* feature. Do *not* use this on your only backup. @@ -2711,7 +2766,7 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to recreate; patterns are supported') - with_lock_epilog = textwrap.dedent(""" + with_lock_epilog = process_epilog(""" This command runs a user-specified command while the repository lock is held. It will first try to acquire the lock (make sure that no other operation is @@ -2747,7 +2802,7 @@ class Archiver: subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', help='additional help on TOPIC') - debug_epilog = textwrap.dedent(""" + debug_epilog = process_epilog(""" These commands are not intended for normal use and potentially very dangerous if used incorrectly. @@ -2764,7 +2819,7 @@ class Archiver: debug_parsers = subparser.add_subparsers(title='required arguments', metavar='') subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser)) - debug_info_epilog = textwrap.dedent(""" + debug_info_epilog = process_epilog(""" This command displays some system information that might be useful for bug reports and debugging problems. If a traceback happens, this information is already appended at the end of the traceback. @@ -2776,7 +2831,7 @@ class Archiver: help='show system infos for debugging / bug reports (debug)') subparser.set_defaults(func=self.do_debug_info) - debug_dump_archive_items_epilog = textwrap.dedent(""" + debug_dump_archive_items_epilog = process_epilog(""" This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. """) subparser = debug_parsers.add_parser('dump-archive-items', parents=[common_parser], add_help=False, @@ -2789,7 +2844,7 @@ class Archiver: type=location_validator(archive=True), help='archive to dump') - debug_dump_archive_epilog = textwrap.dedent(""" + debug_dump_archive_epilog = process_epilog(""" This command dumps all metadata of an archive in a decoded form to a file. """) subparser = debug_parsers.add_parser('dump-archive', parents=[common_parser], add_help=False, @@ -2804,7 +2859,7 @@ class Archiver: subparser.add_argument('path', metavar='PATH', type=str, help='file to dump data into') - debug_dump_manifest_epilog = textwrap.dedent(""" + debug_dump_manifest_epilog = process_epilog(""" This command dumps manifest metadata of a repository in a decoded form to a file. """) subparser = debug_parsers.add_parser('dump-manifest', parents=[common_parser], add_help=False, @@ -2819,7 +2874,7 @@ class Archiver: subparser.add_argument('path', metavar='PATH', type=str, help='file to dump data into') - debug_dump_repo_objs_epilog = textwrap.dedent(""" + debug_dump_repo_objs_epilog = process_epilog(""" This command dumps raw (but decrypted and decompressed) repo objects to files. """) subparser = debug_parsers.add_parser('dump-repo-objs', parents=[common_parser], add_help=False, @@ -2832,7 +2887,7 @@ class Archiver: type=location_validator(archive=False), help='repo to dump') - debug_get_obj_epilog = textwrap.dedent(""" + debug_get_obj_epilog = process_epilog(""" This command gets an object from the repository. """) subparser = debug_parsers.add_parser('get-obj', parents=[common_parser], add_help=False, @@ -2849,7 +2904,7 @@ class Archiver: subparser.add_argument('path', metavar='PATH', type=str, help='file to write object data into') - debug_put_obj_epilog = textwrap.dedent(""" + debug_put_obj_epilog = process_epilog(""" This command puts objects into the repository. """) subparser = debug_parsers.add_parser('put-obj', parents=[common_parser], add_help=False, @@ -2864,7 +2919,7 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, help='file(s) to read and create object(s) from') - debug_delete_obj_epilog = textwrap.dedent(""" + debug_delete_obj_epilog = process_epilog(""" This command deletes objects from the repository. """) subparser = debug_parsers.add_parser('delete-obj', parents=[common_parser], add_help=False, @@ -2879,7 +2934,7 @@ class Archiver: subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, help='hex object ID(s) to delete from the repo') - debug_refcount_obj_epilog = textwrap.dedent(""" + debug_refcount_obj_epilog = process_epilog(""" This command displays the reference count for objects from the repository. """) subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], add_help=False, From 15dfaae223d8ba52bbbddf9979244ef729e89922 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 5 Feb 2017 21:17:42 +0100 Subject: [PATCH 03/10] docs: create: move --exclude note to main doc --- docs/usage.rst | 8 -------- src/borg/archiver.py | 13 +++++++++---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index bdc6b23df..b00f157b0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -437,14 +437,6 @@ Examples # As above, but add nanoseconds $ borg create /path/to/repo::{hostname}-{user}-{now:%Y-%m-%dT%H:%M:%S.%f} ~ -Notes -~~~~~ - -- the --exclude patterns are not like tar. In tar --exclude .bundler/gems will - exclude foo/.bundler/gems. In borg it will not, you need to use --exclude - '\*/.bundler/gems' to get the same effect. See ``borg help patterns`` for - more information. - .. include:: usage/extract.rst.inc diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 65d39bd90..feca6ccfe 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2037,6 +2037,11 @@ class Archiver: .. man NOTES + The --exclude patterns are not like tar. In tar --exclude .bundler/gems will + exclude foo/.bundler/gems. In borg it will not, you need to use --exclude + '\*/.bundler/gems' to get the same effect. See ``borg help patterns`` for + more information. + Item flags ++++++++++ @@ -2115,12 +2120,12 @@ class Archiver: 'http://www.brynosaurus.com/cachedir/spec.html)') exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present', metavar='NAME', action='append', type=str, - help='exclude directories that are tagged by containing a filesystem object with \ - the given NAME') + help='exclude directories that are tagged by containing a filesystem object with ' + 'the given NAME') exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags', action='store_true', default=False, - help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \ - excluded caches/directories') + help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise ' + 'excluded caches/directories') fs_group = subparser.add_argument_group('Filesystem options') fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', From fa24e1f38fcc87f33b17dceb51d974185bd2803b Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 5 Feb 2017 21:32:24 +0100 Subject: [PATCH 04/10] man pages: add borg(1) master/intro page --- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 13 +- docs/man/borg-delete.1 | 2 +- docs/man/borg-diff.1 | 2 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 21 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 47 ++ docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- docs/man/borg.1 | 567 ++++++++++++++++++++++ docs/man_intro.rst | 68 +++ docs/quickstart.rst | 63 +-- docs/quickstart_example.rst.inc | 62 +++ docs/usage.rst | 335 +------------ docs/usage/create.rst.inc | 62 ++- docs/usage/general.rst.inc | 329 +++++++++++++ docs/usage/init.rst.inc | 19 +- docs/usage/key_change-passphrase.rst.inc | 22 + docs/usage/key_migrate-to-repokey.rst.inc | 36 ++ docs/usage/list.rst.inc | 7 +- docs/usage/prune.rst.inc | 2 +- docs/usage/recreate.rst.inc | 16 +- setup.py | 12 +- 40 files changed, 1264 insertions(+), 463 deletions(-) create mode 100644 docs/man/borg-key.1 create mode 100644 docs/man/borg.1 create mode 100644 docs/man_intro.rst create mode 100644 docs/quickstart_example.rst.inc create mode 100644 docs/usage/general.rst.inc create mode 100644 docs/usage/key_change-passphrase.rst.inc create mode 100644 docs/usage/key_migrate-to-repokey.rst.inc diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index b5870e563..b8162436b 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-02-05" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index fb32bfa20..b74dd336b 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 6608887bc..8a0ec36bf 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-02-05" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index 17afcff95..e480c1f8d 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-02-05" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index cc5fbafd7..51124fea3 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-02-05" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 3e30a343e..d01d77e27 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -100,10 +100,10 @@ read exclude patterns from EXCLUDEFILE, one per line exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) .TP .BI \-\-exclude\-if\-present \ NAME -exclude directories that are tagged by containing a filesystem object with the given NAME +exclude directories that are tagged by containing a filesystem object with the given NAME .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -keep tag objects (i.e.: arguments to \-\-exclude\-if\-present) in otherwise excluded caches/directories +keep tag objects (i.e.: arguments to \-\-exclude\-if\-present) in otherwise excluded caches/directories .UNINDENT .SS Filesystem options .INDENT 0.0 @@ -216,15 +216,12 @@ $ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S.%f} ~ .fi .UNINDENT .UNINDENT -.SS Notes -.INDENT 0.0 -.IP \(bu 2 -the \-\-exclude patterns are not like tar. In tar \-\-exclude .bundler/gems will +.SH NOTES +.sp +The \-\-exclude patterns are not like tar. In tar \-\-exclude .bundler/gems will exclude foo/.bundler/gems. In borg it will not, you need to use \-\-exclude \(aq*/.bundler/gems\(aq to get the same effect. See \fBborg help patterns\fP for more information. -.UNINDENT -.SH NOTES .SS Item flags .sp \fB\-\-list\fP outputs a list of all files, directories and other diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index e3a73797a..5b0ec5296 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index dc3d9be99..f488a72ac 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-02-05" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 2770a5142..426be31e0 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-02-05" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 2609912df..305f18fad 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-02-05" "" "borg backup tool" +.TH BORG-INFO 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 40d8a25dd..bbe020c43 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-02-05" "" "borg backup tool" +.TH BORG-INIT 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . @@ -82,32 +82,33 @@ You can change your passphrase for existing repos at any time, it won\(aqt affec the encryption/decryption key or other secrets. .SS Encryption modes .sp -repokey and keyfile use AES\-CTR\-256 for encryption and HMAC\-SHA256 for +\fIrepokey\fP and \fIkeyfile\fP use AES\-CTR\-256 for encryption and HMAC\-SHA256 for authentication in an encrypt\-then\-MAC (EtM) construction. The chunk ID hash is HMAC\-SHA256 as well (with a separate key). These modes are compatible with borg 1.0.x. .sp -repokey\-blake2 and keyfile\-blake2 are also authenticated encryption modes, +\fIrepokey\-blake2\fP and \fIkeyfile\-blake2\fP are also authenticated encryption modes, but use BLAKE2b\-256 instead of HMAC\-SHA256 for authentication. The chunk ID hash is a keyed BLAKE2b\-256 hash. -These modes are new and not compatible with borg 1.0.x. +These modes are new and \fInot\fP compatible with borg 1.0.x. .sp -"authenticated" mode uses no encryption, but authenticates repository contents +\fIauthenticated\fP mode uses no encryption, but authenticates repository contents through the same keyed BLAKE2b\-256 hash as the other blake2 modes (it uses it as chunk ID hash). The key is stored like repokey. This mode is new and not compatible with borg 1.0.x. .sp -"none" mode uses no encryption and no authentication. It uses sha256 as chunk +\fInone\fP mode uses no encryption and no authentication. It uses sha256 as chunk ID hash. Not recommended, rather consider using an authenticated or authenticated/encrypted mode. This mode is compatible with borg 1.0.x. .sp Hardware acceleration will be used automatically. .sp -On modern Intel/AMD CPUs (except very cheap ones), AES is usually hw -accelerated. BLAKE2b is faster than sha256 on Intel/AMD 64bit CPUs. +On modern Intel/AMD CPUs (except very cheap ones), AES is usually +hardware\-accelerated. BLAKE2b is faster than SHA256 on Intel/AMD 64bit CPUs, +which makes \fIauthenticated\fP faster than \fInone\fP\&. .sp -On modern ARM CPUs, NEON provides hw acceleration for sha256 making it faster +On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster than BLAKE2b\-256 there. .SH OPTIONS .sp @@ -122,7 +123,7 @@ repository to create .INDENT 0.0 .TP .B \-e\fP,\fB \-\-encryption -select encryption key mode (default: "None") +select encryption key mode .TP .B \-a\fP,\fB \-\-append\-only create an append\-only mode repository diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 63a58b0c5..86a30ebfa 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 3a8158146..672023032 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-02-05" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 7215df92a..91ce569dd 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-02-05" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index c1c5e95f1..774c9199b 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-02-05" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 new file mode 100644 index 000000000..2d82f003c --- /dev/null +++ b/docs/man/borg-key.1 @@ -0,0 +1,47 @@ +.\" Man page generated from reStructuredText. +. +.TH BORG-KEY 1 "2017-02-11" "" "borg backup tool" +.SH NAME +borg-key \- Manage a keyfile or repokey of a repository +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.nf +borg key export ... +borg key import ... +borg key change\-passphrase ... +borg key migrate\-to\-repokey ... +.fi +.sp +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP, \fIborg\-key\-export(1)\fP, \fIborg\-key\-import(1)\fP, \fIborg\-key\-change\-passphrase(1)\fP, \fIborg\-key\-migrate\-to\-repokey(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index e788bdbae..63a12ceec 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-02-05" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index f814edc3c..646694bfc 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-02-05" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index bff600a45..49cfb6dc7 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-02-05" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 81756e66d..ff8afa1d5 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 55b01cc82..db822491d 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index ee568f2f5..e3c1ee773 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-02-05" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 673ae2696..ee62ee7f8 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 4de127a5c..7d8af5878 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-02-05" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index d329fa885..0c0df0d59 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-02-05" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index d6e19958b..2db9c3fc8 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-02-05" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-02-11" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . diff --git a/docs/man/borg.1 b/docs/man/borg.1 new file mode 100644 index 000000000..b720d71c2 --- /dev/null +++ b/docs/man/borg.1 @@ -0,0 +1,567 @@ +.\" Man page generated from reStructuredText. +. +.TH BORG 1 "2017-02-05" "" "borg backup tool" +.SH NAME +borg \- deduplicating and encrypting backup tool +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.sp +borg [options] [arguments] +.SH DESCRIPTION +.\" we don't include the README.rst here since we want to keep this terse. +. +.sp +BorgBackup (short: Borg) is a deduplicating backup program. +Optionally, it supports compression and authenticated encryption. +.sp +The main goal of Borg is to provide an efficient and secure way to backup data. +The data deduplication technique used makes Borg suitable for daily backups +since only changes are stored. +The authenticated encryption technique makes it suitable for backups to not +fully trusted targets. +.sp +Borg stores a set of files in an \fIarchive\fP\&. A \fIrepository\fP is a collection +of \fIarchives\fP\&. The format of repositories is Borg\-specific. Borg does not +distinguish archives from each other in a any way other than their name, +it does not matter when or where archives where created (eg. different hosts). +.SH EXAMPLES +.SS A step\-by\-step example +.INDENT 0.0 +.IP 1. 3 +Before a backup can be made a repository has to be initialized: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg init \-\-encryption=repokey /path/to/repo +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 2. 3 +Backup the \fB~/src\fP and \fB~/Documents\fP directories into an archive called +\fIMonday\fP: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg create /path/to/repo::Monday ~/src ~/Documents +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 3. 3 +The next day create a new archive called \fITuesday\fP: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg create \-\-stats /path/to/repo::Tuesday ~/src ~/Documents +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This backup will be a lot quicker and a lot smaller since only new never +before seen data is stored. The \fB\-\-stats\fP option causes Borg to +output statistics about the newly created archive such as the amount of unique +data (not shared with other archives): +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- +Archive name: Tuesday +Archive fingerprint: bd31004d58f51ea06ff735d2e5ac49376901b21d58035f8fb05dbf866566e3c2 +Time (start): Tue, 2016\-02\-16 18:15:11 +Time (end): Tue, 2016\-02\-16 18:15:11 + +Duration: 0.19 seconds +Number of files: 127 +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- + Original size Compressed size Deduplicated size +This archive: 4.16 MB 4.17 MB 26.78 kB +All archives: 8.33 MB 8.34 MB 4.19 MB + + Unique chunks Total chunks +Chunk index: 132 261 +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 4. 3 +List all archives in the repository: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg list /path/to/repo +Monday Mon, 2016\-02\-15 19:14:44 +Tuesday Tue, 2016\-02\-16 19:15:11 +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 5. 3 +List the contents of the \fIMonday\fP archive: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg list /path/to/repo::Monday +drwxr\-xr\-x user group 0 Mon, 2016\-02\-15 18:22:30 home/user/Documents +\-rw\-r\-\-r\-\- user group 7961 Mon, 2016\-02\-15 18:22:30 home/user/Documents/Important.doc +\&... +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 6. 3 +Restore the \fIMonday\fP archive by extracting the files relative to the current directory: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg extract /path/to/repo::Monday +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 7. 3 +Recover disk space by manually deleting the \fIMonday\fP archive: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg delete /path/to/repo::Monday +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Borg is quiet by default (it works on WARNING log level). +You can use options like \fB\-\-progress\fP or \fB\-\-list\fP to get specific +reports during command execution. You can also add the \fB\-v\fP (or +\fB\-\-verbose\fP or \fB\-\-info\fP) option to adjust the log level to INFO to +get other informational messages. +.UNINDENT +.UNINDENT +.SH NOTES +.SS Repository URLs +.sp +\fBLocal filesystem\fP (or locally mounted network filesystem): +.sp +\fB/path/to/repo\fP \- filesystem path to repo directory, absolute path +.sp +\fBpath/to/repo\fP \- filesystem path to repo directory, relative path +.sp +Also, stuff like \fB~/path/to/repo\fP or \fB~other/path/to/repo\fP works (this is +expanded by your shell). +.sp +Note: you may also prepend a \fBfile://\fP to a filesystem path to get URL style. +.sp +\fBRemote repositories\fP accessed via ssh \fI\%user@host\fP: +.sp +\fBuser@host:/path/to/repo\fP \- remote repo, absolute path +.sp +\fBssh://user@host:port/path/to/repo\fP \- same, alternative syntax, port can be given +.sp +\fBRemote repositories with relative pathes\fP can be given using this syntax: +.sp +\fBuser@host:path/to/repo\fP \- path relative to current directory +.sp +\fBuser@host:~/path/to/repo\fP \- path relative to user\(aqs home directory +.sp +\fBuser@host:~other/path/to/repo\fP \- path relative to other\(aqs home directory +.sp +Note: giving \fBuser@host:/./path/to/repo\fP or \fBuser@host:/~/path/to/repo\fP or +\fBuser@host:/~other/path/to/repo\fP is also supported, but not required here. +.sp +\fBRemote repositories with relative pathes, alternative syntax with port\fP: +.sp +\fBssh://user@host:port/./path/to/repo\fP \- path relative to current directory +.sp +\fBssh://user@host:port/~/path/to/repo\fP \- path relative to user\(aqs home directory +.sp +\fBssh://user@host:port/~other/path/to/repo\fP \- path relative to other\(aqs home directory +.sp +If you frequently need the same repo URL, it is a good idea to set the +\fBBORG_REPO\fP environment variable to set a default for the repo URL: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +export BORG_REPO=\(aqssh://user@host:port/path/to/repo\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Then just leave away the repo URL if only a repo URL is needed and you want +to use the default \- it will be read from BORG_REPO then. +.sp +Use \fB::\fP syntax to give the repo URL when syntax requires giving a positional +argument for the repo (e.g. \fBborg mount :: /mnt\fP). +.SS Repository / Archive Locations +.sp +Many commands want either a repository (just give the repo URL, see above) or +an archive location, which is a repo URL followed by \fB::archive_name\fP\&. +.sp +Archive names must not contain the \fB/\fP (slash) character. For simplicity, +maybe also avoid blanks or other characters that have special meaning on the +shell or in a filesystem (borg mount will use the archive name as directory +name). +.sp +If you have set BORG_REPO (see above) and an archive location is needed, use +\fB::archive_name\fP \- the repo URL part is then read from BORG_REPO. +.SS Type of log output +.sp +The log level of the builtin logging configuration defaults to WARNING. +This is because we want Borg to be mostly silent and only output +warnings, errors and critical messages, unless output has been requested +by supplying an option that implies output (eg, \-\-list or \-\-progress). +.sp +Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL +.sp +Use \fB\-\-debug\fP to set DEBUG log level \- +to get debug, info, warning, error and critical level output. +.sp +Use \fB\-\-info\fP (or \fB\-v\fP or \fB\-\-verbose\fP) to set INFO log level \- +to get info, warning, error and critical level output. +.sp +Use \fB\-\-warning\fP (default) to set WARNING log level \- +to get warning, error and critical level output. +.sp +Use \fB\-\-error\fP to set ERROR log level \- +to get error and critical level output. +.sp +Use \fB\-\-critical\fP to set CRITICAL log level \- +to get critical level output. +.sp +While you can set misc. log levels, do not expect that every command will +give different output on different log levels \- it\(aqs just a possibility. +.sp +\fBWARNING:\fP +.INDENT 0.0 +.INDENT 3.5 +Options \-\-critical and \-\-error are provided for completeness, +their usage is not recommended as you might miss important information. +.UNINDENT +.UNINDENT +.SS Return codes +.sp +Borg can exit with the following return codes (rc): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +0 = success (logged as INFO) +1 = warning (operation reached its normal end, but there were warnings \- + you should check the log, logged as WARNING) +2 = error (like a fatal error, a local or remote exception, the operation + did not reach its normal end, logged as ERROR) +128+N = killed by signal N (e.g. 137 == kill \-9) +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +If you use \fB\-\-show\-rc\fP, the return code is also logged at the indicated +level as the last log entry. +.SS Environment Variables +.sp +Borg uses some environment variables for automation: +.INDENT 0.0 +.TP +.B General: +.INDENT 7.0 +.TP +.B BORG_REPO +When set, use the value to give the default repository location. If a command needs an archive +parameter, you can abbreviate as \fI::archive\fP\&. If a command needs a repository parameter, you +can either leave it away or abbreviate as \fI::\fP, if a positional parameter is required. +.TP +.B BORG_PASSPHRASE +When set, use the value to answer the passphrase question for encrypted repositories. +It is used when a passphrase is needed to access a encrypted repo as well as when a new +passphrase should be initially set when initializing an encrypted repo. +See also BORG_NEW_PASSPHRASE. +.TP +.B BORG_NEW_PASSPHRASE +When set, use the value to answer the passphrase question when a \fBnew\fP passphrase is asked for. +This variable is checked first. If it is not set, BORG_PASSPHRASE will be checked also. +Main usecase for this is to fully automate \fBborg change\-passphrase\fP\&. +.TP +.B BORG_DISPLAY_PASSPHRASE +When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories. +.TP +.B BORG_LOGGING_CONF +When set, use the given filename as \fI\%INI\fP\-style logging configuration. +.TP +.B BORG_RSH +When set, use this command instead of \fBssh\fP\&. This can be used to specify ssh options, such as +a custom identity file \fBssh \-i /path/to/private/key\fP\&. See \fBman ssh\fP for other options. +.TP +.B BORG_REMOTE_PATH +When set, use the given path/filename as remote path (default is "borg"). +Using \fB\-\-remote\-path PATH\fP commandline option overrides the environment variable. +.TP +.B BORG_FILES_CACHE_TTL +When set to a numeric value, this determines the maximum "time to live" for the files cache +entries (default: 20). The files cache is used to quickly determine whether a file is unchanged. +The FAQ explains this more detailed in: \fIalways_chunking\fP +.TP +.B TMPDIR +where temporary files are stored (might need a lot of temporary space for some operations) +.UNINDENT +.TP +.B Some automatic "answerers" (if set, they automatically answer confirmation questions): +.INDENT 7.0 +.TP +.B BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes) +For "Warning: Attempting to access a previously unknown unencrypted repository" +.TP +.B BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes) +For "Warning: The repository at location ... was previously located at ..." +.TP +.B BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) +For "Warning: \(aqcheck \-\-repair\(aq is an experimental feature that might result in data loss." +.TP +.B BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) +For "You requested to completely DELETE the repository \fIincluding\fP all archives it contains:" +.TP +.B BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) +For "recreate is an experimental feature." +.UNINDENT +.sp +Note: answers are case sensitive. setting an invalid answer value might either give the default +answer or ask you interactively, depending on whether retries are allowed (they by default are +allowed). So please test your scripts interactively before making them a non\-interactive script. +.TP +.B Directories and files: +.INDENT 7.0 +.TP +.B BORG_KEYS_DIR +Default to \(aq~/.config/borg/keys\(aq. This directory contains keys for encrypted repositories. +.TP +.B BORG_KEY_FILE +When set, use the given filename as repository key file. +.TP +.B BORG_SECURITY_DIR +Default to \(aq~/.config/borg/security\(aq. This directory contains information borg uses to +track its usage of NONCES ("numbers used once" \- usually in encryption context) and other +security relevant data. +.TP +.B BORG_CACHE_DIR +Default to \(aq~/.cache/borg\(aq. This directory contains the local cache and might need a lot +of space for dealing with big repositories). +.UNINDENT +.TP +.B Building: +.INDENT 7.0 +.TP +.B BORG_OPENSSL_PREFIX +Adds given OpenSSL header file directory to the default locations (setup.py). +.TP +.B BORG_LZ4_PREFIX +Adds given LZ4 header file directory to the default locations (setup.py). +.TP +.B BORG_LIBB2_PREFIX +Adds given prefix directory to the default locations. If a \(aqinclude/blake2.h\(aq is found Borg +will be linked against the system libb2 instead of a bundled implementation. (setup.py) +.UNINDENT +.UNINDENT +.sp +Please note: +.INDENT 0.0 +.IP \(bu 2 +be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data\(aqs security/safety +.IP \(bu 2 +also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions +(e.g. mode 600, root:root). +.UNINDENT +.SS File systems +.sp +We strongly recommend against using Borg (or any other database\-like +software) on non\-journaling file systems like FAT, since it is not +possible to assume any consistency in case of power failures (or a +sudden disconnect of an external drive or similar failures). +.sp +While Borg uses a data store that is resilient against these failures +when used on journaling file systems, it is not possible to guarantee +this with some hardware \-\- independent of the software used. We don\(aqt +know a list of affected hardware. +.sp +If you are suspicious whether your Borg repository is still consistent +and readable after one of the failures mentioned above occured, run +\fBborg check \-\-verify\-data\fP to make sure it is consistent. +.SS Units +.sp +To display quantities, Borg takes care of respecting the +usual conventions of scale. Disk sizes are displayed in \fI\%decimal\fP, using powers of ten (so +\fBkB\fP means 1000 bytes). For memory usage, \fI\%binary prefixes\fP are used, and are +indicated using the \fI\%IEC binary prefixes\fP, +using powers of two (so \fBKiB\fP means 1024 bytes). +.SS Date and Time +.sp +We format date and time conforming to ISO\-8601, that is: YYYY\-MM\-DD and +HH:MM:SS (24h clock). +.sp +For more information about that, see: \fI\%https://xkcd.com/1179/\fP +.sp +Unless otherwise noted, we display local date and time. +Internally, we store and process date and time as UTC. +.SS Resource Usage +.sp +Borg might use a lot of resources depending on the size of the data set it is dealing with. +.sp +If one uses Borg in a client/server way (with a ssh: repository), +the resource usage occurs in part on the client and in another part on the +server. +.sp +If one uses Borg as a single process (with a filesystem repo), +all the resource usage occurs in that one process, so just add up client + +server to get the approximate resource usage. +.INDENT 0.0 +.TP +.B CPU client: +borg create: does chunking, hashing, compression, crypto (high CPU usage) +chunks cache sync: quite heavy on CPU, doing lots of hashtable operations. +borg extract: crypto, decompression (medium to high CPU usage) +borg check: similar to extract, but depends on options given. +borg prune / borg delete archive: low to medium CPU usage +borg delete repo: done on the server +It won\(aqt go beyond 100% of 1 core as the code is currently single\-threaded. +Especially higher zlib and lzma compression levels use significant amounts +of CPU cycles. Crypto might be cheap on the CPU (if hardware accelerated) or +expensive (if not). +.TP +.B CPU server: +It usually doesn\(aqt need much CPU, it just deals with the key/value store +(repository) and uses the repository index for that. +.sp +borg check: the repository check computes the checksums of all chunks +(medium CPU usage) +borg delete repo: low CPU usage +.TP +.B CPU (only for client/server operation): +When using borg in a client/server way with a \fI\%ssh:\-type\fP repo, the ssh +processes used for the transport layer will need some CPU on the client and +on the server due to the crypto they are doing \- esp. if you are pumping +big amounts of data. +.TP +.B Memory (RAM) client: +The chunks index and the files index are read into memory for performance +reasons. Might need big amounts of memory (see below). +Compression, esp. lzma compression with high levels might need substantial +amounts of memory. +.TP +.B Memory (RAM) server: +The server process will load the repository index into memory. Might need +considerable amounts of memory, but less than on the client (see below). +.TP +.B Chunks index (client only): +Proportional to the amount of data chunks in your repo. Lots of chunks +in your repo imply a big chunks index. +It is possible to tweak the chunker params (see create options). +.TP +.B Files index (client only): +Proportional to the amount of files in your last backups. Can be switched +off (see create options), but next backup might be much slower if you do. +The speed benefit of using the files cache is proportional to file size. +.TP +.B Repository index (server only): +Proportional to the amount of data chunks in your repo. Lots of chunks +in your repo imply a big repository index. +It is possible to tweak the chunker params (see create options) to +influence the amount of chunks being created. +.TP +.B Temporary files (client): +Reading data and metadata from a FUSE mounted repository will consume up to +the size of all deduplicated, small chunks in the repository. Big chunks +won\(aqt be locally cached. +.TP +.B Temporary files (server): +None. +.TP +.B Cache files (client only): +Contains the chunks index and files index (plus a collection of single\- +archive chunk indexes which might need huge amounts of disk space, +depending on archive count and size \- see FAQ about how to reduce). +.TP +.B Network (only for client/server operation): +If your repository is remote, all deduplicated (and optionally compressed/ +encrypted) data of course has to go over the connection (ssh: repo url). +If you use a locally mounted network filesystem, additionally some copy +operations used for transaction support also go over the connection. If +you backup multiple sources to one target repository, additional traffic +happens for cache resynchronization. +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP for common command line options +.sp +\fIborg\-init(1)\fP, +\fIborg\-create(1)\fP, \fIborg\-mount(1)\fP, \fIborg\-extract(1)\fP, +\fIborg\-list(1)\fP, \fIborg\-info(1)\fP, +\fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, +\fIborg\-recreate(1)\fP +.sp +\fIborg\-compression(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP +.INDENT 0.0 +.IP \(bu 2 +Main web site \fI\%https://borgbackup.readthedocs.org/\fP +.IP \(bu 2 +Releases \fI\%https://github.com/borgbackup/borg/releases\fP +.IP \(bu 2 +Changelog \fI\%https://github.com/borgbackup/borg/blob/master/docs/changes.rst\fP +.IP \(bu 2 +GitHub \fI\%https://github.com/borgbackup/borg\fP +.IP \(bu 2 +Security contact \fI\%https://borgbackup.readthedocs.io/en/latest/support.html#security\-contact\fP +.UNINDENT +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man_intro.rst b/docs/man_intro.rst new file mode 100644 index 000000000..0e08f5a99 --- /dev/null +++ b/docs/man_intro.rst @@ -0,0 +1,68 @@ +==== +borg +==== + +---------------------------------------- +deduplicating and encrypting backup tool +---------------------------------------- + +:Author: The Borg Collective +:Date: 2017-02-05 +:Manual section: 1 +:Manual group: borg backup tool + +SYNOPSIS +-------- + +borg [options] [arguments] + +DESCRIPTION +----------- + +.. we don't include the README.rst here since we want to keep this terse. + +BorgBackup (short: Borg) is a deduplicating backup program. +Optionally, it supports compression and authenticated encryption. + +The main goal of Borg is to provide an efficient and secure way to backup data. +The data deduplication technique used makes Borg suitable for daily backups +since only changes are stored. +The authenticated encryption technique makes it suitable for backups to not +fully trusted targets. + +Borg stores a set of files in an *archive*. A *repository* is a collection +of *archives*. The format of repositories is Borg-specific. Borg does not +distinguish archives from each other in a any way other than their name, +it does not matter when or where archives where created (eg. different hosts). + +EXAMPLES +-------- + +A step-by-step example +~~~~~~~~~~~~~~~~~~~~~~ + +.. include:: quickstart_example.rst.inc + +NOTES +----- + +.. include:: usage/general.rst.inc + +SEE ALSO +-------- + +`borg-common(1)` for common command line options + +`borg-init(1)`, +`borg-create(1)`, `borg-mount(1)`, `borg-extract(1)`, +`borg-list(1)`, `borg-info(1)`, +`borg-delete(1)`, `borg-prune(1)`, +`borg-recreate(1)` + +`borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)` + +* Main web site https://borgbackup.readthedocs.org/ +* Releases https://github.com/borgbackup/borg/releases +* Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst +* GitHub https://github.com/borgbackup/borg +* Security contact https://borgbackup.readthedocs.io/en/latest/support.html#security-contact diff --git a/docs/quickstart.rst b/docs/quickstart.rst index f8835f29f..32770c2cb 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -11,68 +11,7 @@ various use cases. A step by step example ---------------------- -1. Before a backup can be made a repository has to be initialized:: - - $ borg init /path/to/repo - -2. Backup the ``~/src`` and ``~/Documents`` directories into an archive called - *Monday*:: - - $ borg create /path/to/repo::Monday ~/src ~/Documents - -3. The next day create a new archive called *Tuesday*:: - - $ borg create --stats /path/to/repo::Tuesday ~/src ~/Documents - - This backup will be a lot quicker and a lot smaller since only new never - before seen data is stored. The ``--stats`` option causes |project_name| to - output statistics about the newly created archive such as the amount of unique - data (not shared with other archives):: - - ------------------------------------------------------------------------------ - Archive name: Tuesday - Archive fingerprint: bd31004d58f51ea06ff735d2e5ac49376901b21d58035f8fb05dbf866566e3c2 - Time (start): Tue, 2016-02-16 18:15:11 - Time (end): Tue, 2016-02-16 18:15:11 - - Duration: 0.19 seconds - Number of files: 127 - ------------------------------------------------------------------------------ - Original size Compressed size Deduplicated size - This archive: 4.16 MB 4.17 MB 26.78 kB - All archives: 8.33 MB 8.34 MB 4.19 MB - - Unique chunks Total chunks - Chunk index: 132 261 - ------------------------------------------------------------------------------ - -4. List all archives in the repository:: - - $ borg list /path/to/repo - Monday Mon, 2016-02-15 19:14:44 - Tuesday Tue, 2016-02-16 19:15:11 - -5. List the contents of the *Monday* archive:: - - $ borg list /path/to/repo::Monday - drwxr-xr-x user group 0 Mon, 2016-02-15 18:22:30 home/user/Documents - -rw-r--r-- user group 7961 Mon, 2016-02-15 18:22:30 home/user/Documents/Important.doc - ... - -6. Restore the *Monday* archive by extracting the files relative to the current directory:: - - $ borg extract /path/to/repo::Monday - -7. Recover disk space by manually deleting the *Monday* archive:: - - $ borg delete /path/to/repo::Monday - -.. Note:: - Borg is quiet by default (it works on WARNING log level). - You can use options like ``--progress`` or ``--list`` to get specific - reports during command execution. You can also add the ``-v`` (or - ``--verbose`` or ``--info``) option to adjust the log level to INFO to - get other informational messages. +.. include:: quickstart_example.rst.inc Important note about free space ------------------------------- diff --git a/docs/quickstart_example.rst.inc b/docs/quickstart_example.rst.inc new file mode 100644 index 000000000..69ed98454 --- /dev/null +++ b/docs/quickstart_example.rst.inc @@ -0,0 +1,62 @@ +1. Before a backup can be made a repository has to be initialized:: + + $ borg init --encryption=repokey /path/to/repo + +2. Backup the ``~/src`` and ``~/Documents`` directories into an archive called + *Monday*:: + + $ borg create /path/to/repo::Monday ~/src ~/Documents + +3. The next day create a new archive called *Tuesday*:: + + $ borg create --stats /path/to/repo::Tuesday ~/src ~/Documents + + This backup will be a lot quicker and a lot smaller since only new never + before seen data is stored. The ``--stats`` option causes Borg to + output statistics about the newly created archive such as the amount of unique + data (not shared with other archives):: + + ------------------------------------------------------------------------------ + Archive name: Tuesday + Archive fingerprint: bd31004d58f51ea06ff735d2e5ac49376901b21d58035f8fb05dbf866566e3c2 + Time (start): Tue, 2016-02-16 18:15:11 + Time (end): Tue, 2016-02-16 18:15:11 + + Duration: 0.19 seconds + Number of files: 127 + ------------------------------------------------------------------------------ + Original size Compressed size Deduplicated size + This archive: 4.16 MB 4.17 MB 26.78 kB + All archives: 8.33 MB 8.34 MB 4.19 MB + + Unique chunks Total chunks + Chunk index: 132 261 + ------------------------------------------------------------------------------ + +4. List all archives in the repository:: + + $ borg list /path/to/repo + Monday Mon, 2016-02-15 19:14:44 + Tuesday Tue, 2016-02-16 19:15:11 + +5. List the contents of the *Monday* archive:: + + $ borg list /path/to/repo::Monday + drwxr-xr-x user group 0 Mon, 2016-02-15 18:22:30 home/user/Documents + -rw-r--r-- user group 7961 Mon, 2016-02-15 18:22:30 home/user/Documents/Important.doc + ... + +6. Restore the *Monday* archive by extracting the files relative to the current directory:: + + $ borg extract /path/to/repo::Monday + +7. Recover disk space by manually deleting the *Monday* archive:: + + $ borg delete /path/to/repo::Monday + +.. Note:: + Borg is quiet by default (it works on WARNING log level). + You can use options like ``--progress`` or ``--list`` to get specific + reports during command execution. You can also add the ``-v`` (or + ``--verbose`` or ``--info``) option to adjust the log level to INFO to + get other informational messages. diff --git a/docs/usage.rst b/docs/usage.rst index b00f157b0..cd43c65e3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -12,342 +12,13 @@ command in detail. General ------- -Repository URLs -~~~~~~~~~~~~~~~ - -**Local filesystem** (or locally mounted network filesystem): - -``/path/to/repo`` - filesystem path to repo directory, absolute path - -``path/to/repo`` - filesystem path to repo directory, relative path - -Also, stuff like ``~/path/to/repo`` or ``~other/path/to/repo`` works (this is -expanded by your shell). - -Note: you may also prepend a ``file://`` to a filesystem path to get URL style. - -**Remote repositories** accessed via ssh user@host: - -``user@host:/path/to/repo`` - remote repo, absolute path - -``ssh://user@host:port/path/to/repo`` - same, alternative syntax, port can be given - - -**Remote repositories with relative pathes** can be given using this syntax: - -``user@host:path/to/repo`` - path relative to current directory - -``user@host:~/path/to/repo`` - path relative to user's home directory - -``user@host:~other/path/to/repo`` - path relative to other's home directory - -Note: giving ``user@host:/./path/to/repo`` or ``user@host:/~/path/to/repo`` or -``user@host:/~other/path/to/repo`` is also supported, but not required here. - - -**Remote repositories with relative pathes, alternative syntax with port**: - -``ssh://user@host:port/./path/to/repo`` - path relative to current directory - -``ssh://user@host:port/~/path/to/repo`` - path relative to user's home directory - -``ssh://user@host:port/~other/path/to/repo`` - path relative to other's home directory - - -If you frequently need the same repo URL, it is a good idea to set the -``BORG_REPO`` environment variable to set a default for the repo URL: - -:: - - export BORG_REPO='ssh://user@host:port/path/to/repo' - -Then just leave away the repo URL if only a repo URL is needed and you want -to use the default - it will be read from BORG_REPO then. - -Use ``::`` syntax to give the repo URL when syntax requires giving a positional -argument for the repo (e.g. ``borg mount :: /mnt``). - - -Repository / Archive Locations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Many commands want either a repository (just give the repo URL, see above) or -an archive location, which is a repo URL followed by ``::archive_name``. - -Archive names must not contain the ``/`` (slash) character. For simplicity, -maybe also avoid blanks or other characters that have special meaning on the -shell or in a filesystem (borg mount will use the archive name as directory -name). - -If you have set BORG_REPO (see above) and an archive location is needed, use -``::archive_name`` - the repo URL part is then read from BORG_REPO. - - -Type of log output -~~~~~~~~~~~~~~~~~~ - -The log level of the builtin logging configuration defaults to WARNING. -This is because we want |project_name| to be mostly silent and only output -warnings, errors and critical messages, unless output has been requested -by supplying an option that implies output (eg, --list or --progress). - -Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL - -Use ``--debug`` to set DEBUG log level - -to get debug, info, warning, error and critical level output. - -Use ``--info`` (or ``-v`` or ``--verbose``) to set INFO log level - -to get info, warning, error and critical level output. - -Use ``--warning`` (default) to set WARNING log level - -to get warning, error and critical level output. - -Use ``--error`` to set ERROR log level - -to get error and critical level output. - -Use ``--critical`` to set CRITICAL log level - -to get critical level output. - -While you can set misc. log levels, do not expect that every command will -give different output on different log levels - it's just a possibility. - -.. warning:: Options --critical and --error are provided for completeness, - their usage is not recommended as you might miss important information. - -Return codes -~~~~~~~~~~~~ - -|project_name| can exit with the following return codes (rc): - -:: - - 0 = success (logged as INFO) - 1 = warning (operation reached its normal end, but there were warnings - - you should check the log, logged as WARNING) - 2 = error (like a fatal error, a local or remote exception, the operation - did not reach its normal end, logged as ERROR) - 128+N = killed by signal N (e.g. 137 == kill -9) - -If you use ``--show-rc``, the return code is also logged at the indicated -level as the last log entry. - - -Environment Variables -~~~~~~~~~~~~~~~~~~~~~ - -|project_name| uses some environment variables for automation: - -General: - BORG_REPO - When set, use the value to give the default repository location. If a command needs an archive - parameter, you can abbreviate as `::archive`. If a command needs a repository parameter, you - can either leave it away or abbreviate as `::`, if a positional parameter is required. - BORG_PASSPHRASE - When set, use the value to answer the passphrase question for encrypted repositories. - It is used when a passphrase is needed to access a encrypted repo as well as when a new - passphrase should be initially set when initializing an encrypted repo. - See also BORG_NEW_PASSPHRASE. - BORG_NEW_PASSPHRASE - When set, use the value to answer the passphrase question when a **new** passphrase is asked for. - This variable is checked first. If it is not set, BORG_PASSPHRASE will be checked also. - Main usecase for this is to fully automate ``borg change-passphrase``. - BORG_DISPLAY_PASSPHRASE - When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories. - BORG_LOGGING_CONF - When set, use the given filename as INI_-style logging configuration. - BORG_RSH - When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as - a custom identity file ``ssh -i /path/to/private/key``. See ``man ssh`` for other options. - BORG_REMOTE_PATH - When set, use the given path/filename as remote path (default is "borg"). - Using ``--remote-path PATH`` commandline option overrides the environment variable. - BORG_FILES_CACHE_TTL - When set to a numeric value, this determines the maximum "time to live" for the files cache - entries (default: 20). The files cache is used to quickly determine whether a file is unchanged. - The FAQ explains this more detailed in: :ref:`always_chunking` - TMPDIR - where temporary files are stored (might need a lot of temporary space for some operations) - -Some automatic "answerers" (if set, they automatically answer confirmation questions): - BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes) - For "Warning: Attempting to access a previously unknown unencrypted repository" - BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes) - For "Warning: The repository at location ... was previously located at ..." - BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) - For "Warning: 'check --repair' is an experimental feature that might result in data loss." - BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) - For "You requested to completely DELETE the repository *including* all archives it contains:" - BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) - For "recreate is an experimental feature." - - Note: answers are case sensitive. setting an invalid answer value might either give the default - answer or ask you interactively, depending on whether retries are allowed (they by default are - allowed). So please test your scripts interactively before making them a non-interactive script. - -Directories and files: - BORG_KEYS_DIR - Default to '~/.config/borg/keys'. This directory contains keys for encrypted repositories. - BORG_KEY_FILE - When set, use the given filename as repository key file. - BORG_SECURITY_DIR - Default to '~/.config/borg/security'. This directory contains information borg uses to - track its usage of NONCES ("numbers used once" - usually in encryption context) and other - security relevant data. - BORG_CACHE_DIR - Default to '~/.cache/borg'. This directory contains the local cache and might need a lot - of space for dealing with big repositories). - -Building: - BORG_OPENSSL_PREFIX - Adds given OpenSSL header file directory to the default locations (setup.py). - BORG_LZ4_PREFIX - Adds given LZ4 header file directory to the default locations (setup.py). - BORG_LIBB2_PREFIX - Adds given prefix directory to the default locations. If a 'include/blake2.h' is found Borg - will be linked against the system libb2 instead of a bundled implementation. (setup.py) - - -Please note: - -- be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data's security/safety -- also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions - (e.g. mode 600, root:root). - - -.. _INI: https://docs.python.org/3.4/library/logging.config.html#configuration-file-format - -Resource Usage -~~~~~~~~~~~~~~ - -|project_name| might use a lot of resources depending on the size of the data set it is dealing with. - -If one uses |project_name| in a client/server way (with a ssh: repository), -the resource usage occurs in part on the client and in another part on the -server. - -If one uses |project_name| as a single process (with a filesystem repo), -all the resource usage occurs in that one process, so just add up client + -server to get the approximate resource usage. - -CPU client: - borg create: does chunking, hashing, compression, crypto (high CPU usage) - chunks cache sync: quite heavy on CPU, doing lots of hashtable operations. - borg extract: crypto, decompression (medium to high CPU usage) - borg check: similar to extract, but depends on options given. - borg prune / borg delete archive: low to medium CPU usage - borg delete repo: done on the server - It won't go beyond 100% of 1 core as the code is currently single-threaded. - Especially higher zlib and lzma compression levels use significant amounts - of CPU cycles. Crypto might be cheap on the CPU (if hardware accelerated) or - expensive (if not). - -CPU server: - It usually doesn't need much CPU, it just deals with the key/value store - (repository) and uses the repository index for that. - - borg check: the repository check computes the checksums of all chunks - (medium CPU usage) - borg delete repo: low CPU usage - -CPU (only for client/server operation): - When using borg in a client/server way with a ssh:-type repo, the ssh - processes used for the transport layer will need some CPU on the client and - on the server due to the crypto they are doing - esp. if you are pumping - big amounts of data. - -Memory (RAM) client: - The chunks index and the files index are read into memory for performance - reasons. Might need big amounts of memory (see below). - Compression, esp. lzma compression with high levels might need substantial - amounts of memory. - -Memory (RAM) server: - The server process will load the repository index into memory. Might need - considerable amounts of memory, but less than on the client (see below). - -Chunks index (client only): - Proportional to the amount of data chunks in your repo. Lots of chunks - in your repo imply a big chunks index. - It is possible to tweak the chunker params (see create options). - -Files index (client only): - Proportional to the amount of files in your last backups. Can be switched - off (see create options), but next backup might be much slower if you do. - The speed benefit of using the files cache is proportional to file size. - -Repository index (server only): - Proportional to the amount of data chunks in your repo. Lots of chunks - in your repo imply a big repository index. - It is possible to tweak the chunker params (see create options) to - influence the amount of chunks being created. - -Temporary files (client): - Reading data and metadata from a FUSE mounted repository will consume up to - the size of all deduplicated, small chunks in the repository. Big chunks - won't be locally cached. - -Temporary files (server): - None. - -Cache files (client only): - Contains the chunks index and files index (plus a collection of single- - archive chunk indexes which might need huge amounts of disk space, - depending on archive count and size - see FAQ about how to reduce). - -Network (only for client/server operation): - If your repository is remote, all deduplicated (and optionally compressed/ - encrypted) data of course has to go over the connection (ssh: repo url). - If you use a locally mounted network filesystem, additionally some copy - operations used for transaction support also go over the connection. If - you backup multiple sources to one target repository, additional traffic - happens for cache resynchronization. +.. include:: usage/general.rst.inc In case you are interested in more details (like formulas), please see :ref:`internals`. -File systems -~~~~~~~~~~~~ - -We strongly recommend against using Borg (or any other database-like -software) on non-journaling file systems like FAT, since it is not -possible to assume any consistency in case of power failures (or a -sudden disconnect of an external drive or similar failures). - -While Borg uses a data store that is resilient against these failures -when used on journaling file systems, it is not possible to guarantee -this with some hardware -- independent of the software used. We don't -know a list of affected hardware. - -If you are suspicious whether your Borg repository is still consistent -and readable after one of the failures mentioned above occured, run -``borg check --verify-data`` to make sure it is consistent. - -Units -~~~~~ - -To display quantities, |project_name| takes care of respecting the -usual conventions of scale. Disk sizes are displayed in `decimal -`_, using powers of ten (so -``kB`` means 1000 bytes). For memory usage, `binary prefixes -`_ are used, and are -indicated using the `IEC binary prefixes -`_, -using powers of two (so ``KiB`` means 1024 bytes). - - -Date and Time -~~~~~~~~~~~~~ - -We format date and time conforming to ISO-8601, that is: YYYY-MM-DD and -HH:MM:SS (24h clock). - -For more information about that, see: https://xkcd.com/1179/ - -Unless otherwise noted, we display local date and time. -Internally, we store and process date and time as UTC. - Common options -~~~~~~~~~~~~~~ +++++++++++++++ All |project_name| commands share these options: @@ -636,6 +307,7 @@ Examples Examples ~~~~~~~~ + borg mount ++++++++++ :: @@ -845,6 +517,7 @@ Here are misc. notes about topics that are maybe not covered in enough detail in --chunker-params ~~~~~~~~~~~~~~~~ + The chunker params influence how input files are cut into pieces (chunks) which are then considered for deduplication. They also have a big impact on resource usage (RAM and disk space) as the amount of resources needed is diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 83c8810eb..6b34278d1 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -36,10 +36,10 @@ Exclusion options | read exclude patterns from EXCLUDEFILE, one per line ``--exclude-caches`` | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) - ``--exclude-if-present FILENAME`` - | exclude directories that contain the specified file - ``--keep-tag-files`` - | keep tag files of excluded caches/directories + ``--exclude-if-present NAME`` + | exclude directories that are tagged by containing a filesystem object with the given NAME + ``--keep-exclude-tags``, ``--keep-tag-files`` + | keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise excluded caches/directories Filesystem options ``-x``, ``--one-file-system`` @@ -58,12 +58,12 @@ Filesystem options Archive options ``--comment COMMENT`` | add a comment text to the archive - ``--timestamp yyyy-mm-ddThh:mm:ss`` - | manually specify the archive creation date/time (UTC). alternatively, give a reference file/directory. + ``--timestamp TIMESTAMP`` + | manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). alternatively, give a reference file/directory. ``-c SECONDS``, ``--checkpoint-interval SECONDS`` | write checkpoint every SECONDS seconds (Default: 1800) - ``--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE`` - | specify the chunker parameters. default: 19,23,21,4095 + ``--chunker-params PARAMS`` + | specify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: 19,23,21,4095 ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm, see the output of the "borg help compression" command for details. ``--compression-from COMPRESSIONCONFIG`` @@ -94,3 +94,49 @@ all files on these file systems. See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. + +.. man NOTES + +The --exclude patterns are not like tar. In tar --exclude .bundler/gems will +exclude foo/.bundler/gems. In borg it will not, you need to use --exclude +'\*/.bundler/gems' to get the same effect. See ``borg help patterns`` for +more information. + +Item flags +++++++++++ + +``--list`` outputs a list of all files, directories and other +file system items it considered (no matter whether they had content changes +or not). For each item, it prefixes a single-letter flag that indicates type +and/or status of the item. + +If you are interested only in a subset of that output, you can give e.g. +``--filter=AME`` and it will only show regular files with A, M or E status (see +below). + +A uppercase character represents the status of a regular file relative to the +"files" cache (not relative to the repo -- this is an issue if the files cache +is not used). Metadata is stored in any case and for 'A' and 'M' also new data +chunks are stored. For 'U' all data chunks refer to already existing chunks. + +- 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ) +- 'M' = regular file, modified +- 'U' = regular file, unchanged +- 'E' = regular file, an error happened while accessing/reading *this* file + +A lowercase character means a file type other than a regular file, +borg usually just stores their metadata: + +- 'd' = directory +- 'b' = block device +- 'c' = char device +- 'h' = regular file, hardlink (to already seen inodes) +- 's' = symlink +- 'f' = fifo + +Other flags used include: + +- 'i' = backup data was read from standard input (stdin) +- '-' = dry run, item was *not* backed up +- 'x' = excluded, item was *not* backed up +- '?' = missing status code (if you see this, please file a bug report!) \ No newline at end of file diff --git a/docs/usage/general.rst.inc b/docs/usage/general.rst.inc new file mode 100644 index 000000000..7cbe7cae1 --- /dev/null +++ b/docs/usage/general.rst.inc @@ -0,0 +1,329 @@ +Repository URLs +~~~~~~~~~~~~~~~ + +**Local filesystem** (or locally mounted network filesystem): + +``/path/to/repo`` - filesystem path to repo directory, absolute path + +``path/to/repo`` - filesystem path to repo directory, relative path + +Also, stuff like ``~/path/to/repo`` or ``~other/path/to/repo`` works (this is +expanded by your shell). + +Note: you may also prepend a ``file://`` to a filesystem path to get URL style. + +**Remote repositories** accessed via ssh user@host: + +``user@host:/path/to/repo`` - remote repo, absolute path + +``ssh://user@host:port/path/to/repo`` - same, alternative syntax, port can be given + + +**Remote repositories with relative pathes** can be given using this syntax: + +``user@host:path/to/repo`` - path relative to current directory + +``user@host:~/path/to/repo`` - path relative to user's home directory + +``user@host:~other/path/to/repo`` - path relative to other's home directory + +Note: giving ``user@host:/./path/to/repo`` or ``user@host:/~/path/to/repo`` or +``user@host:/~other/path/to/repo`` is also supported, but not required here. + + +**Remote repositories with relative pathes, alternative syntax with port**: + +``ssh://user@host:port/./path/to/repo`` - path relative to current directory + +``ssh://user@host:port/~/path/to/repo`` - path relative to user's home directory + +``ssh://user@host:port/~other/path/to/repo`` - path relative to other's home directory + + +If you frequently need the same repo URL, it is a good idea to set the +``BORG_REPO`` environment variable to set a default for the repo URL: + +:: + + export BORG_REPO='ssh://user@host:port/path/to/repo' + +Then just leave away the repo URL if only a repo URL is needed and you want +to use the default - it will be read from BORG_REPO then. + +Use ``::`` syntax to give the repo URL when syntax requires giving a positional +argument for the repo (e.g. ``borg mount :: /mnt``). + + +Repository / Archive Locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many commands want either a repository (just give the repo URL, see above) or +an archive location, which is a repo URL followed by ``::archive_name``. + +Archive names must not contain the ``/`` (slash) character. For simplicity, +maybe also avoid blanks or other characters that have special meaning on the +shell or in a filesystem (borg mount will use the archive name as directory +name). + +If you have set BORG_REPO (see above) and an archive location is needed, use +``::archive_name`` - the repo URL part is then read from BORG_REPO. + + +Type of log output +~~~~~~~~~~~~~~~~~~ + +The log level of the builtin logging configuration defaults to WARNING. +This is because we want Borg to be mostly silent and only output +warnings, errors and critical messages, unless output has been requested +by supplying an option that implies output (eg, --list or --progress). + +Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL + +Use ``--debug`` to set DEBUG log level - +to get debug, info, warning, error and critical level output. + +Use ``--info`` (or ``-v`` or ``--verbose``) to set INFO log level - +to get info, warning, error and critical level output. + +Use ``--warning`` (default) to set WARNING log level - +to get warning, error and critical level output. + +Use ``--error`` to set ERROR log level - +to get error and critical level output. + +Use ``--critical`` to set CRITICAL log level - +to get critical level output. + +While you can set misc. log levels, do not expect that every command will +give different output on different log levels - it's just a possibility. + +.. warning:: Options --critical and --error are provided for completeness, + their usage is not recommended as you might miss important information. + +Return codes +~~~~~~~~~~~~ + +Borg can exit with the following return codes (rc): + +:: + + 0 = success (logged as INFO) + 1 = warning (operation reached its normal end, but there were warnings - + you should check the log, logged as WARNING) + 2 = error (like a fatal error, a local or remote exception, the operation + did not reach its normal end, logged as ERROR) + 128+N = killed by signal N (e.g. 137 == kill -9) + +If you use ``--show-rc``, the return code is also logged at the indicated +level as the last log entry. + + +Environment Variables +~~~~~~~~~~~~~~~~~~~~~ + +Borg uses some environment variables for automation: + +General: + BORG_REPO + When set, use the value to give the default repository location. If a command needs an archive + parameter, you can abbreviate as `::archive`. If a command needs a repository parameter, you + can either leave it away or abbreviate as `::`, if a positional parameter is required. + BORG_PASSPHRASE + When set, use the value to answer the passphrase question for encrypted repositories. + It is used when a passphrase is needed to access a encrypted repo as well as when a new + passphrase should be initially set when initializing an encrypted repo. + See also BORG_NEW_PASSPHRASE. + BORG_NEW_PASSPHRASE + When set, use the value to answer the passphrase question when a **new** passphrase is asked for. + This variable is checked first. If it is not set, BORG_PASSPHRASE will be checked also. + Main usecase for this is to fully automate ``borg change-passphrase``. + BORG_DISPLAY_PASSPHRASE + When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories. + BORG_LOGGING_CONF + When set, use the given filename as INI_-style logging configuration. + BORG_RSH + When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as + a custom identity file ``ssh -i /path/to/private/key``. See ``man ssh`` for other options. + BORG_REMOTE_PATH + When set, use the given path/filename as remote path (default is "borg"). + Using ``--remote-path PATH`` commandline option overrides the environment variable. + BORG_FILES_CACHE_TTL + When set to a numeric value, this determines the maximum "time to live" for the files cache + entries (default: 20). The files cache is used to quickly determine whether a file is unchanged. + The FAQ explains this more detailed in: :ref:`always_chunking` + TMPDIR + where temporary files are stored (might need a lot of temporary space for some operations) + +Some automatic "answerers" (if set, they automatically answer confirmation questions): + BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes) + For "Warning: Attempting to access a previously unknown unencrypted repository" + BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes) + For "Warning: The repository at location ... was previously located at ..." + BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) + For "Warning: 'check --repair' is an experimental feature that might result in data loss." + BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) + For "You requested to completely DELETE the repository *including* all archives it contains:" + BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) + For "recreate is an experimental feature." + + Note: answers are case sensitive. setting an invalid answer value might either give the default + answer or ask you interactively, depending on whether retries are allowed (they by default are + allowed). So please test your scripts interactively before making them a non-interactive script. + +Directories and files: + BORG_KEYS_DIR + Default to '~/.config/borg/keys'. This directory contains keys for encrypted repositories. + BORG_KEY_FILE + When set, use the given filename as repository key file. + BORG_SECURITY_DIR + Default to '~/.config/borg/security'. This directory contains information borg uses to + track its usage of NONCES ("numbers used once" - usually in encryption context) and other + security relevant data. + BORG_CACHE_DIR + Default to '~/.cache/borg'. This directory contains the local cache and might need a lot + of space for dealing with big repositories). + +Building: + BORG_OPENSSL_PREFIX + Adds given OpenSSL header file directory to the default locations (setup.py). + BORG_LZ4_PREFIX + Adds given LZ4 header file directory to the default locations (setup.py). + BORG_LIBB2_PREFIX + Adds given prefix directory to the default locations. If a 'include/blake2.h' is found Borg + will be linked against the system libb2 instead of a bundled implementation. (setup.py) + + +Please note: + +- be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data's security/safety +- also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions + (e.g. mode 600, root:root). + + +.. _INI: https://docs.python.org/3.4/library/logging.config.html#configuration-file-format + +File systems +~~~~~~~~~~~~ + +We strongly recommend against using Borg (or any other database-like +software) on non-journaling file systems like FAT, since it is not +possible to assume any consistency in case of power failures (or a +sudden disconnect of an external drive or similar failures). + +While Borg uses a data store that is resilient against these failures +when used on journaling file systems, it is not possible to guarantee +this with some hardware -- independent of the software used. We don't +know a list of affected hardware. + +If you are suspicious whether your Borg repository is still consistent +and readable after one of the failures mentioned above occured, run +``borg check --verify-data`` to make sure it is consistent. + +Units +~~~~~ + +To display quantities, Borg takes care of respecting the +usual conventions of scale. Disk sizes are displayed in `decimal +`_, using powers of ten (so +``kB`` means 1000 bytes). For memory usage, `binary prefixes +`_ are used, and are +indicated using the `IEC binary prefixes +`_, +using powers of two (so ``KiB`` means 1024 bytes). + +Date and Time +~~~~~~~~~~~~~ + +We format date and time conforming to ISO-8601, that is: YYYY-MM-DD and +HH:MM:SS (24h clock). + +For more information about that, see: https://xkcd.com/1179/ + +Unless otherwise noted, we display local date and time. +Internally, we store and process date and time as UTC. + +Resource Usage +~~~~~~~~~~~~~~ + +Borg might use a lot of resources depending on the size of the data set it is dealing with. + +If one uses Borg in a client/server way (with a ssh: repository), +the resource usage occurs in part on the client and in another part on the +server. + +If one uses Borg as a single process (with a filesystem repo), +all the resource usage occurs in that one process, so just add up client + +server to get the approximate resource usage. + +CPU client: + borg create: does chunking, hashing, compression, crypto (high CPU usage) + chunks cache sync: quite heavy on CPU, doing lots of hashtable operations. + borg extract: crypto, decompression (medium to high CPU usage) + borg check: similar to extract, but depends on options given. + borg prune / borg delete archive: low to medium CPU usage + borg delete repo: done on the server + It won't go beyond 100% of 1 core as the code is currently single-threaded. + Especially higher zlib and lzma compression levels use significant amounts + of CPU cycles. Crypto might be cheap on the CPU (if hardware accelerated) or + expensive (if not). + +CPU server: + It usually doesn't need much CPU, it just deals with the key/value store + (repository) and uses the repository index for that. + + borg check: the repository check computes the checksums of all chunks + (medium CPU usage) + borg delete repo: low CPU usage + +CPU (only for client/server operation): + When using borg in a client/server way with a ssh:-type repo, the ssh + processes used for the transport layer will need some CPU on the client and + on the server due to the crypto they are doing - esp. if you are pumping + big amounts of data. + +Memory (RAM) client: + The chunks index and the files index are read into memory for performance + reasons. Might need big amounts of memory (see below). + Compression, esp. lzma compression with high levels might need substantial + amounts of memory. + +Memory (RAM) server: + The server process will load the repository index into memory. Might need + considerable amounts of memory, but less than on the client (see below). + +Chunks index (client only): + Proportional to the amount of data chunks in your repo. Lots of chunks + in your repo imply a big chunks index. + It is possible to tweak the chunker params (see create options). + +Files index (client only): + Proportional to the amount of files in your last backups. Can be switched + off (see create options), but next backup might be much slower if you do. + The speed benefit of using the files cache is proportional to file size. + +Repository index (server only): + Proportional to the amount of data chunks in your repo. Lots of chunks + in your repo imply a big repository index. + It is possible to tweak the chunker params (see create options) to + influence the amount of chunks being created. + +Temporary files (client): + Reading data and metadata from a FUSE mounted repository will consume up to + the size of all deduplicated, small chunks in the repository. Big chunks + won't be locally cached. + +Temporary files (server): + None. + +Cache files (client only): + Contains the chunks index and files index (plus a collection of single- + archive chunk indexes which might need huge amounts of disk space, + depending on archive count and size - see FAQ about how to reduce). + +Network (only for client/server operation): + If your repository is remote, all deduplicated (and optionally compressed/ + encrypted) data of course has to go over the connection (ssh: repo url). + If you use a locally mounted network filesystem, additionally some copy + operations used for transaction support also go over the connection. If + you backup multiple sources to one target repository, additional traffic + happens for cache resynchronization. diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index c8633b237..9fb3264a8 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -14,7 +14,7 @@ positional arguments optional arguments ``-e``, ``--encryption`` - | select encryption key mode (default: "None") + | select encryption key mode ``-a``, ``--append-only`` | create an append-only mode repository @@ -70,30 +70,31 @@ the encryption/decryption key or other secrets. Encryption modes ++++++++++++++++ -repokey and keyfile use AES-CTR-256 for encryption and HMAC-SHA256 for +`repokey` and `keyfile` use AES-CTR-256 for encryption and HMAC-SHA256 for authentication in an encrypt-then-MAC (EtM) construction. The chunk ID hash is HMAC-SHA256 as well (with a separate key). These modes are compatible with borg 1.0.x. -repokey-blake2 and keyfile-blake2 are also authenticated encryption modes, +`repokey-blake2` and `keyfile-blake2` are also authenticated encryption modes, but use BLAKE2b-256 instead of HMAC-SHA256 for authentication. The chunk ID hash is a keyed BLAKE2b-256 hash. -These modes are new and not compatible with borg 1.0.x. +These modes are new and *not* compatible with borg 1.0.x. -"authenticated" mode uses no encryption, but authenticates repository contents +`authenticated` mode uses no encryption, but authenticates repository contents through the same keyed BLAKE2b-256 hash as the other blake2 modes (it uses it as chunk ID hash). The key is stored like repokey. This mode is new and not compatible with borg 1.0.x. -"none" mode uses no encryption and no authentication. It uses sha256 as chunk +`none` mode uses no encryption and no authentication. It uses sha256 as chunk ID hash. Not recommended, rather consider using an authenticated or authenticated/encrypted mode. This mode is compatible with borg 1.0.x. Hardware acceleration will be used automatically. -On modern Intel/AMD CPUs (except very cheap ones), AES is usually hw -accelerated. BLAKE2b is faster than sha256 on Intel/AMD 64bit CPUs. +On modern Intel/AMD CPUs (except very cheap ones), AES is usually +hardware-accelerated. BLAKE2b is faster than SHA256 on Intel/AMD 64bit CPUs, +which makes `authenticated` faster than `none`. -On modern ARM CPUs, NEON provides hw acceleration for sha256 making it faster +On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster than BLAKE2b-256 there. diff --git a/docs/usage/key_change-passphrase.rst.inc b/docs/usage/key_change-passphrase.rst.inc new file mode 100644 index 000000000..cbc0d9e6d --- /dev/null +++ b/docs/usage/key_change-passphrase.rst.inc @@ -0,0 +1,22 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_key_change-passphrase: + +borg key change-passphrase +-------------------------- +:: + + borg key change-passphrase REPOSITORY + +positional arguments + REPOSITORY + + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +The key files used for repository encryption are optionally passphrase +protected. This command can be used to change this passphrase. diff --git a/docs/usage/key_migrate-to-repokey.rst.inc b/docs/usage/key_migrate-to-repokey.rst.inc new file mode 100644 index 000000000..0e82f28c2 --- /dev/null +++ b/docs/usage/key_migrate-to-repokey.rst.inc @@ -0,0 +1,36 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_key_migrate-to-repokey: + +borg key migrate-to-repokey +--------------------------- +:: + + borg key migrate-to-repokey REPOSITORY + +positional arguments + REPOSITORY + + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command migrates a repository from passphrase mode (removed in Borg 1.0) +to repokey mode. + +You will be first asked for the repository passphrase (to open it in passphrase +mode). This is the same passphrase as you used to use for this repo before 1.0. + +It will then derive the different secrets from this passphrase. + +Then you will be asked for a new passphrase (twice, for safety). This +passphrase will be used to protect the repokey (which contains these same +secrets in encrypted form). You may use the same passphrase as you used to +use, but you may also use a different one. + +After migrating to repokey mode, you can change the passphrase at any time. +But please note: the secrets will always stay the same and they could always +be derived from your (old) passphrase-mode passphrase. diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index c848e5924..d3a4485f7 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -46,6 +46,7 @@ This command lists the contents of a repository or an archive. See the "borg help patterns" command for more help on exclude patterns. The following keys are available for --format: + - NEWLINE: OS dependent line separator - NL: alias of NEWLINE - NUL: NUL character for creating print0 / xargs -0 like output, see barchive/bpath @@ -54,13 +55,15 @@ The following keys are available for --format: - CR - LF --- Keys for listing repository archives: +Keys for listing repository archives: + - archive: archive name interpreted as text (might be missing non-text characters, see barchive) - barchive: verbatim archive name, can contain any character except NUL - time: time of creation of the archive - id: internal ID of the archive --- Keys for listing archive files: +Keys for listing archive files: + - type - mode - uid diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index 01c58d1b2..b6a0052c3 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -57,7 +57,7 @@ automated backup scripts wanting to keep a certain number of historic backups. Also, prune automatically removes checkpoint archives (incomplete archives left behind by interrupted backup runs) except if the checkpoint is the latest archive (and thus still needed). Checkpoint archives are not considered when -comparing archive counts against the retention limits (--keep-*). +comparing archive counts against the retention limits (--keep-X). If a prefix is set with -P, then only archives that start with the prefix are considered for deletion and only those archives count towards the totals diff --git a/docs/usage/recreate.rst.inc b/docs/usage/recreate.rst.inc index 02f06a8c6..93f7f414f 100644 --- a/docs/usage/recreate.rst.inc +++ b/docs/usage/recreate.rst.inc @@ -36,10 +36,10 @@ Exclusion options | read exclude patterns from EXCLUDEFILE, one per line ``--exclude-caches`` | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) - ``--exclude-if-present FILENAME`` - | exclude directories that contain the specified file - ``--keep-tag-files`` - | keep tag files of excluded caches/directories + ``--exclude-if-present NAME`` + | exclude directories that are tagged by containing a filesystem object with the given NAME + ``--keep-exclude-tags``, ``--keep-tag-files`` + | keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise excluded caches/directories Archive options ``--target TARGET`` @@ -48,16 +48,16 @@ Archive options | write checkpoint every SECONDS seconds (Default: 1800) ``--comment COMMENT`` | add a comment text to the archive - ``--timestamp yyyy-mm-ddThh:mm:ss`` - | manually specify the archive creation date/time (UTC). alternatively, give a reference file/directory. + ``--timestamp TIMESTAMP`` + | manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). alternatively, give a reference file/directory. ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm, see the output of the "borg help compression" command for details. ``--always-recompress`` | always recompress chunks, don't skip chunks already compressed with the same algorithm. ``--compression-from COMPRESSIONCONFIG`` | read compression patterns from COMPRESSIONCONFIG, see the output of the "borg help compression" command for details. - ``--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE`` - | specify the chunker parameters (or "default"). + ``--chunker-params PARAMS`` + | specify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE) or "default" to use the current defaults. default: 19,23,21,4095 Description ~~~~~~~~~~~ diff --git a/setup.py b/setup.py index 6e57a5eda..a2a5cb8f5 100644 --- a/setup.py +++ b/setup.py @@ -353,6 +353,7 @@ class build_man(Command): .. role:: ref(title) .. |project_name| replace:: Borg + """) def initialize_options(self): @@ -372,6 +373,7 @@ class build_man(Command): self.generate_level('', parser, Archiver) self.build_topic_pages(Archiver) + self.build_intro_page() def generate_level(self, prefix, parser, Archiver): is_subcommand = False @@ -447,6 +449,12 @@ class build_man(Command): write(text) self.gen_man_page(man_title, doc.getvalue()) + def build_intro_page(self): + print('building man page borg(1)', file=sys.stderr) + with open('docs/man_intro.rst') as fd: + man_intro = fd.read() + self.gen_man_page('borg', self.rst_prelude + man_intro) + def new_doc(self): doc = io.StringIO(self.rst_prelude) doc.read() @@ -502,7 +510,9 @@ class build_man(Command): def gen_man_page(self, name, rst): from docutils.writers import manpage from docutils.core import publish_string - man_page = publish_string(source=rst, writer=manpage.Writer()) + # We give the source_path so that docutils can find relative includes + # as-if the document where located in the docs/ directory. + man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer()) with open('docs/man/%s.1' % name, 'wb') as fd: fd.write(man_page) From 1d9378f12ce23ed32783aecddf6881844f490089 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 11 Feb 2017 15:45:09 +0100 Subject: [PATCH 05/10] man pages: generate page for subparsers, eg. borg-key(1) Conflicts: docs/man/borg-key.1 --- docs/man/borg-key.1 | 2 +- setup.py | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 2d82f003c..ca1ec2eb3 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-02-11" "" "borg backup tool" +.TH BORG-KEY 1 "2017-02-12" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/setup.py b/setup.py index a2a5cb8f5..771f11b51 100644 --- a/setup.py +++ b/setup.py @@ -393,27 +393,35 @@ class build_man(Command): man_title = 'borg-' + command.replace(' ', '-') print('building man page', man_title + '(1)', file=sys.stderr) - if self.generate_level(command + ' ', parser, Archiver): - continue + is_intermediary = self.generate_level(command + ' ', parser, Archiver) doc, write = self.new_doc() self.write_man_header(write, man_title, parser.description) self.write_heading(write, 'SYNOPSIS') - write('borg', command, end='') - self.write_usage(write, parser) + if is_intermediary: + subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0] + for subcommand in subparsers.choices: + write('| borg', command, subcommand, '...') + self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand)) + else: + write('borg', command, end='') + self.write_usage(write, parser) write('\n') - self.write_heading(write, 'DESCRIPTION') description, _, notes = parser.epilog.partition('\n.. man NOTES') - write(description) - self.write_heading(write, 'OPTIONS') - write('See `borg-common(1)` for common options of Borg commands.') - write() - self.write_options(write, parser) + if description: + self.write_heading(write, 'DESCRIPTION') + write(description) - self.write_examples(write, command) + if not is_intermediary: + self.write_heading(write, 'OPTIONS') + write('See `borg-common(1)` for common options of Borg commands.') + write() + self.write_options(write, parser) + + self.write_examples(write, command) if notes: self.write_heading(write, 'NOTES') From 96fa1e8414b5cd02ebc4c53b589f68f8bfd6b9fa Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 5 Feb 2017 23:01:41 +0100 Subject: [PATCH 06/10] docs: define "ours" merge strategy for auto-generated files --- .gitattributes | 2 ++ docs/man_intro.rst | 2 +- docs/usage.rst | 2 +- docs/{usage/general.rst.inc => usage_general.rst.inc} | 0 4 files changed, 4 insertions(+), 2 deletions(-) rename docs/{usage/general.rst.inc => usage_general.rst.inc} (100%) diff --git a/.gitattributes b/.gitattributes index 9d00a6907..3724dd9dd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,5 @@ borg/_version.py export-subst *.py diff=python +docs/usage/* merge=ours +docs/man/* merge=ours diff --git a/docs/man_intro.rst b/docs/man_intro.rst index 0e08f5a99..b726e3318 100644 --- a/docs/man_intro.rst +++ b/docs/man_intro.rst @@ -46,7 +46,7 @@ A step-by-step example NOTES ----- -.. include:: usage/general.rst.inc +.. include:: usage_general.rst.inc SEE ALSO -------- diff --git a/docs/usage.rst b/docs/usage.rst index cd43c65e3..2ad55d024 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -12,7 +12,7 @@ command in detail. General ------- -.. include:: usage/general.rst.inc +.. include:: usage_general.rst.inc In case you are interested in more details (like formulas), please see :ref:`internals`. diff --git a/docs/usage/general.rst.inc b/docs/usage_general.rst.inc similarity index 100% rename from docs/usage/general.rst.inc rename to docs/usage_general.rst.inc From 95b69a433c48ba12aed1e612f93946e4d5ceb4fa Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 5 Feb 2017 23:02:43 +0100 Subject: [PATCH 07/10] docs: usage: remove not-updated-anymore debug* files. --- docs/usage/debug-delete-obj.rst.inc | 23 ------------------- docs/usage/debug-dump-archive-items.rst.inc | 21 ----------------- docs/usage/debug-dump-repo-objs.rst.inc | 21 ----------------- docs/usage/debug-get-obj.rst.inc | 25 --------------------- docs/usage/debug-info.rst.inc | 19 ---------------- docs/usage/debug-put-obj.rst.inc | 23 ------------------- docs/usage/debug_delete-obj.rst.inc | 23 ------------------- docs/usage/debug_dump-archive-items.rst.inc | 21 ----------------- docs/usage/debug_dump-repo-objs.rst.inc | 21 ----------------- docs/usage/debug_get-obj.rst.inc | 25 --------------------- docs/usage/debug_info.rst.inc | 19 ---------------- docs/usage/debug_put-obj.rst.inc | 23 ------------------- docs/usage/debug_refcount-obj.rst.inc | 23 ------------------- 13 files changed, 287 deletions(-) delete mode 100644 docs/usage/debug-delete-obj.rst.inc delete mode 100644 docs/usage/debug-dump-archive-items.rst.inc delete mode 100644 docs/usage/debug-dump-repo-objs.rst.inc delete mode 100644 docs/usage/debug-get-obj.rst.inc delete mode 100644 docs/usage/debug-info.rst.inc delete mode 100644 docs/usage/debug-put-obj.rst.inc delete mode 100644 docs/usage/debug_delete-obj.rst.inc delete mode 100644 docs/usage/debug_dump-archive-items.rst.inc delete mode 100644 docs/usage/debug_dump-repo-objs.rst.inc delete mode 100644 docs/usage/debug_get-obj.rst.inc delete mode 100644 docs/usage/debug_info.rst.inc delete mode 100644 docs/usage/debug_put-obj.rst.inc delete mode 100644 docs/usage/debug_refcount-obj.rst.inc diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc deleted file mode 100644 index 4fcfb48f9..000000000 --- a/docs/usage/debug-delete-obj.rst.inc +++ /dev/null @@ -1,23 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-delete-obj: - -borg debug-delete-obj ---------------------- -:: - - borg debug-delete-obj REPOSITORY IDs - -positional arguments - REPOSITORY - repository to use - IDs - hex object ID(s) to delete from the repo - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command deletes objects from the repository. diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc deleted file mode 100644 index 63c395460..000000000 --- a/docs/usage/debug-dump-archive-items.rst.inc +++ /dev/null @@ -1,21 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-dump-archive-items: - -borg debug-dump-archive-items ------------------------------ -:: - - borg debug-dump-archive-items ARCHIVE - -positional arguments - ARCHIVE - archive to dump - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. diff --git a/docs/usage/debug-dump-repo-objs.rst.inc b/docs/usage/debug-dump-repo-objs.rst.inc deleted file mode 100644 index 3910a126d..000000000 --- a/docs/usage/debug-dump-repo-objs.rst.inc +++ /dev/null @@ -1,21 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-dump-repo-objs: - -borg debug-dump-repo-objs -------------------------- -:: - - borg debug-dump-repo-objs REPOSITORY - -positional arguments - REPOSITORY - repo to dump - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command dumps raw (but decrypted and decompressed) repo objects to files. diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc deleted file mode 100644 index a0b3f4577..000000000 --- a/docs/usage/debug-get-obj.rst.inc +++ /dev/null @@ -1,25 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-get-obj: - -borg debug-get-obj ------------------- -:: - - borg debug-get-obj REPOSITORY ID PATH - -positional arguments - REPOSITORY - repository to use - ID - hex object ID to get from the repo - PATH - file to write object data into - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command gets an object from the repository. diff --git a/docs/usage/debug-info.rst.inc b/docs/usage/debug-info.rst.inc deleted file mode 100644 index 4812aaf45..000000000 --- a/docs/usage/debug-info.rst.inc +++ /dev/null @@ -1,19 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-info: - -borg debug-info ---------------- -:: - - borg debug-info - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command displays some system information that might be useful for bug -reports and debugging problems. If a traceback happens, this information is -already appended at the end of the traceback. diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc deleted file mode 100644 index d03ace849..000000000 --- a/docs/usage/debug-put-obj.rst.inc +++ /dev/null @@ -1,23 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-put-obj: - -borg debug-put-obj ------------------- -:: - - borg debug-put-obj REPOSITORY PATH - -positional arguments - REPOSITORY - repository to use - PATH - file(s) to read and create object(s) from - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command puts objects into the repository. diff --git a/docs/usage/debug_delete-obj.rst.inc b/docs/usage/debug_delete-obj.rst.inc deleted file mode 100644 index 71248e1d7..000000000 --- a/docs/usage/debug_delete-obj.rst.inc +++ /dev/null @@ -1,23 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_delete-obj: - -borg debug delete-obj ---------------------- -:: - - borg debug delete-obj REPOSITORY IDs - -positional arguments - REPOSITORY - repository to use - IDs - hex object ID(s) to delete from the repo - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command deletes objects from the repository. diff --git a/docs/usage/debug_dump-archive-items.rst.inc b/docs/usage/debug_dump-archive-items.rst.inc deleted file mode 100644 index a28711536..000000000 --- a/docs/usage/debug_dump-archive-items.rst.inc +++ /dev/null @@ -1,21 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_dump-archive-items: - -borg debug dump-archive-items ------------------------------ -:: - - borg debug dump-archive-items ARCHIVE - -positional arguments - ARCHIVE - archive to dump - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. diff --git a/docs/usage/debug_dump-repo-objs.rst.inc b/docs/usage/debug_dump-repo-objs.rst.inc deleted file mode 100644 index e041ecc18..000000000 --- a/docs/usage/debug_dump-repo-objs.rst.inc +++ /dev/null @@ -1,21 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_dump-repo-objs: - -borg debug dump-repo-objs -------------------------- -:: - - borg debug dump-repo-objs REPOSITORY - -positional arguments - REPOSITORY - repo to dump - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command dumps raw (but decrypted and decompressed) repo objects to files. diff --git a/docs/usage/debug_get-obj.rst.inc b/docs/usage/debug_get-obj.rst.inc deleted file mode 100644 index 8a966314e..000000000 --- a/docs/usage/debug_get-obj.rst.inc +++ /dev/null @@ -1,25 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_get-obj: - -borg debug get-obj ------------------- -:: - - borg debug get-obj REPOSITORY ID PATH - -positional arguments - REPOSITORY - repository to use - ID - hex object ID to get from the repo - PATH - file to write object data into - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command gets an object from the repository. diff --git a/docs/usage/debug_info.rst.inc b/docs/usage/debug_info.rst.inc deleted file mode 100644 index ccfbeb197..000000000 --- a/docs/usage/debug_info.rst.inc +++ /dev/null @@ -1,19 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_info: - -borg debug info ---------------- -:: - - borg debug info - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command displays some system information that might be useful for bug -reports and debugging problems. If a traceback happens, this information is -already appended at the end of the traceback. diff --git a/docs/usage/debug_put-obj.rst.inc b/docs/usage/debug_put-obj.rst.inc deleted file mode 100644 index 5563593bd..000000000 --- a/docs/usage/debug_put-obj.rst.inc +++ /dev/null @@ -1,23 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_put-obj: - -borg debug put-obj ------------------- -:: - - borg debug put-obj REPOSITORY PATH - -positional arguments - REPOSITORY - repository to use - PATH - file(s) to read and create object(s) from - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command puts objects into the repository. diff --git a/docs/usage/debug_refcount-obj.rst.inc b/docs/usage/debug_refcount-obj.rst.inc deleted file mode 100644 index c38fcc7a1..000000000 --- a/docs/usage/debug_refcount-obj.rst.inc +++ /dev/null @@ -1,23 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug_refcount-obj: - -borg debug refcount-obj ------------------------ -:: - - borg debug refcount-obj REPOSITORY IDs - -positional arguments - REPOSITORY - repository to use - IDs - hex object ID(s) to show refcounts for - -`Common options`_ - | - -Description -~~~~~~~~~~~ - -This command displays the reference count for objects from the repository. From 847797b477a4481720c118dede44636f88f019f3 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 11 Feb 2017 16:01:26 +0100 Subject: [PATCH 08/10] update development.rst for docs changes --- docs/development.rst | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index afefed0f8..b567d37a6 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -141,25 +141,36 @@ Important notes: - When using ``--`` to give options to py.test, you MUST also give ``borg.testsuite[.module]``. -Regenerate usage files ----------------------- +Documentation +------------- -Usage and API documentation is currently committed directly to git, -although those files are generated automatically from the source -tree. +Generated files +~~~~~~~~~~~~~~~ -When a new module is added, the ``docs/api.rst`` file needs to be -regenerated:: - - ./setup.py build_api +Usage documentation (found in ``docs/usage/``) and man pages +(``docs/man/``) are generated automatically from the command line +parsers declared in the program and their documentation, which is +embedded in the program (see archiver.py). These are committed to git +for easier use by packagers downstream. When a command is added, a commandline flag changed, added or removed, the usage docs need to be rebuilt as well:: - ./setup.py build_usage + python setup.py build_usage + python setup.py build_man + +However, we prefer to do this as part of our :ref:`releasing` +preparations, so it is generally not necessary to update these when +submitting patches that change something about the command line. + +The code documentation (which is currently not part of the released +docs) also uses a generated file (``docs/api.rst``), that needs to be +updated when a module is added or removed:: + + python setup.py build_api Building the docs with Sphinx ------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The documentation (in reStructuredText format, .rst) is in docs/. @@ -230,6 +241,7 @@ therefore we recommend to use these merge parameters:: git merge 1.0-maint -s recursive -X rename-threshold=20% +.. _releasing: Creating a new release ---------------------- @@ -243,7 +255,7 @@ Checklist: - update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..`` - check version number of upcoming release in ``CHANGES.rst`` - verify that ``MANIFEST.in`` and ``setup.py`` are complete -- ``python setup.py build_api ; python setup.py build_usage`` and commit +- ``python setup.py build_api ; python setup.py build_usage ; python setup.py build_man`` and commit - tag the release:: git tag -s -m "tagged/signed release X.Y.Z" X.Y.Z From 0971fdce8d14f337ce3f635ebd6ac78711edec21 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 11 Feb 2017 16:05:56 +0100 Subject: [PATCH 09/10] docs: changes: fix two rST warnings --- docs/changes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 084a42d6b..3c317ef4f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -232,7 +232,7 @@ Version 1.0.10rc1 (2017-01-29) Bug fixes: - borg serve: fix transmission data loss of pipe writes, #1268 - This affects only the cygwin platform (not Linux, *BSD, OS X). + This affects only the cygwin platform (not Linux, BSD, OS X). - Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992 - When running out of buffer memory when reading xattrs, only skip the current file, #1993 @@ -285,7 +285,7 @@ Other changes: - remove .github from pypi package, #2051 - add pip and setuptools to requirements file, #2030 - SyncFile: fix use of fd object after close (cosmetic) -- Manifest.in: simplify, exclude *.{so,dll,orig}, #2066 +- Manifest.in: simplify, exclude \*.{so,dll,orig}, #2066 - ignore posix_fadvise errors in repository.py, #2095 (works around issues with docker on ARM) - make LoggedIO.close_segment reentrant, avoid reentrance From e4486cd370a276d029429edd619d58c397787eab Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 11 Feb 2017 16:07:55 +0100 Subject: [PATCH 10/10] fix rST warning in Repository.scan --- src/borg/repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/borg/repository.py b/src/borg/repository.py index 9d4d604dd..4183473e2 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -788,6 +788,7 @@ class Repository: fetching data in this order does linear reads and reuses stuff from disk cache. We rely on repository.check() has run already (either now or some time before) and that: + - if we are called from a borg check command, self.index is a valid, fresh, in-sync repo index. - if we are called from elsewhere, either self.index or the on-disk index is valid and in-sync. - the repository segments are valid (no CRC errors).