diff --git a/Vagrantfile b/Vagrantfile index 0a6d9452f..6fdf2727c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -176,19 +176,19 @@ def install_source_dependencies(user) make -j$(nproc) install PREFIX=${PREFIX} # libzstd - VERSION_LIBZSTD=1.5.4 + VERSION_LIBZSTD=1.5.5 git -C ${PREFIX}/src clone --depth 1 --branch v${VERSION_LIBZSTD} https://github.com/facebook/zstd.git cd ${PREFIX}/src/zstd make -j$(nproc) install PREFIX=${PREFIX} # xxHash - VERSION_LIBXXHASH=0.8.1 + VERSION_LIBXXHASH=0.8.2 git -C ${PREFIX}/src clone --depth 1 --branch v${VERSION_LIBXXHASH} https://github.com/Cyan4973/xxHash.git cd ${PREFIX}/src/xxHash make -j$(nproc) install PREFIX=${PREFIX} # openssl - VERSION_OPENSSL=1_1_1t + VERSION_OPENSSL=1_1_1w git -C ${PREFIX}/src clone --depth 1 --branch OpenSSL_${VERSION_OPENSSL} https://github.com/openssl/openssl.git cd ${PREFIX}/src/openssl ./config --prefix=${PREFIX} --openssldir=${PREFIX}/lib/ssl @@ -208,7 +208,7 @@ def install_source_dependencies(user) ln -s ${PREFIX}/src/meson/meson.py ${PREFIX}/bin/meson # libfuse3 - VERSION_LIBFUSE=3.14.0 + VERSION_LIBFUSE=3.16.1 git -C ${PREFIX}/src clone --depth 1 --branch fuse-${VERSION_LIBFUSE} https://github.com/libfuse/libfuse.git cd ${PREFIX}/src/libfuse mkdir build; cd build @@ -243,7 +243,7 @@ def install_pythons(boxname) return <<-EOF . ~/.bash_profile echo "PYTHON_CONFIGURE_OPTS: ${PYTHON_CONFIGURE_OPTS}" - pyenv install 3.11.2 # tests, binary build + pyenv install 3.11.5 # tests, binary build pyenv install 3.10.2 # tests pyenv install 3.9.10 # tests pyenv rehash @@ -263,8 +263,8 @@ def build_pyenv_venv(boxname) . ~/.bash_profile cd /vagrant/borg # use the latest 3.11 release - pyenv global 3.11.2 - pyenv virtualenv 3.11.2 borg-env + pyenv global 3.11.5 + pyenv virtualenv 3.11.5 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end @@ -288,7 +288,7 @@ def install_pyinstaller() . ~/.bash_profile cd /vagrant/borg . borg-env/bin/activate - pip install 'pyinstaller==5.13.1' + pip install 'pyinstaller==5.13.2' EOF end @@ -311,8 +311,8 @@ def run_tests(boxname, skip_env) . ../borg-env/bin/activate if which pyenv 2> /dev/null; then # for testing, use the earliest point releases of the supported python versions: - pyenv global 3.9.10 3.10.2 3.11.2 - pyenv local 3.9.10 3.10.2 3.11.2 + pyenv global 3.9.10 3.10.2 3.11.5 + pyenv local 3.9.10 3.10.2 3.11.5 fi # otherwise: just use the system python # some OSes can only run specific test envs, e.g. because they miss FUSE support: diff --git a/docs/changes.rst b/docs/changes.rst index 3641035f3..e73507d02 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -120,8 +120,8 @@ Compatibility notes: Change Log 2.x ============== -Version 2.0.0b7 (not released yet) ----------------------------------- +Version 2.0.0b7 (2023-09-14) +---------------------------- Please note: @@ -148,6 +148,8 @@ Other changes: - always implicitly require archive TAMs (all archives have TAMs since borg 1.2.6) - always implicitly require manifest TAMs (manifests have TAMs since borg 1.0.9) - rlist: remove support for {tam} placeholder, archives are now always TAM-authenticated. +- support / test on Python 3.12 +- allow msgpack 1.0.6 (which has py312 wheels), #7810 - manifest: move item_keys into config dict (manifest.version == 2 now), #7710 - replace "datetime.utcfromtimestamp" to avoid deprecation warnings with Python 3.12 - properly normalise paths on Windows (forward slashes, integrate drive letter into path) diff --git a/docs/man/borg-benchmark-cpu.1 b/docs/man/borg-benchmark-cpu.1 index 6a221e43b..52eea6a24 100644 --- a/docs/man/borg-benchmark-cpu.1 +++ b/docs/man/borg-benchmark-cpu.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK-CPU" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-BENCHMARK-CPU" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-benchmark-cpu \- Benchmark CPU bound operations. .SH SYNOPSIS diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index f62c0b6a4..4def48ef0 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK-CRUD" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-BENCHMARK-CRUD" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. .SH SYNOPSIS diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 14092a348..27314f65f 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-BENCHMARK" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command .SH SYNOPSIS diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index a48c4f993..a5621c3b7 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BREAK-LOCK" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-BREAK-LOCK" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. .SH SYNOPSIS diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 80c0f0a73..a4dc3822a 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CHECK" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-CHECK" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency .SH SYNOPSIS @@ -35,94 +35,123 @@ borg-check \- Check repository consistency borg [common options] check [options] .SH DESCRIPTION .sp -The check command verifies the consistency of a repository and the corresponding archives. +The check command verifies the consistency of a repository and its archives. +It consists of two major steps: +.INDENT 0.0 +.IP 1. 3 +Checking the consistency of the repository itself. This includes checking +the segment magic headers, and both the metadata and data of all objects in +the segments. The read data is checked by size and CRC. Bit rot and other +types of accidental damage can be detected this way. Running the repository +check can be split into multiple partial checks using \fB\-\-max\-duration\fP\&. +When checking a remote repository, please note that the checks run on the +server and do not cause significant network traffic. +.IP 2. 3 +Checking consistency and correctness of the archive metadata and optionally +archive data (requires \fB\-\-verify\-data\fP). This includes ensuring that the +repository manifest exists, the archive metadata chunk is present, and that +all chunks referencing files (items) in the archive exist. This requires +reading archive and file metadata, but not data. To cryptographically verify +the file (content) data integrity pass \fB\-\-verify\-data\fP, but keep in mind +that this requires reading all data and is hence very time consuming. When +checking archives of a remote repository, archive checks run on the client +machine because they require decrypting data and therefore the encryption +key. +.UNINDENT .sp -check \-\-repair is a potentially dangerous function and might lead to data loss -(for kinds of corruption it is not capable of dealing with). BE VERY CAREFUL! +Both steps can also be run independently. Pass \fB\-\-repository\-only\fP to run the +repository checks only, or pass \fB\-\-archives\-only\fP to run the archive checks +only. +.sp +The \fB\-\-max\-duration\fP option can be used to split a long\-running repository +check into multiple partial checks. After the given number of seconds the check +is interrupted. The next partial check will continue where the previous one +stopped, until the full repository has been checked. Assuming a complete check +would take 7 hours, then running a daily check with \fB\-\-max\-duration=3600\fP +(1 hour) would result in one full repository check per week. Doing a full +repository check aborts any previous partial check; the next partial check will +restart from the beginning. With partial repository checks you can run neither +archive checks, nor enable repair mode. Consequently, if you want to use +\fB\-\-max\-duration\fP you must also pass \fB\-\-repository\-only\fP, and must not pass +\fB\-\-archives\-only\fP, nor \fB\-\-repair\fP\&. +.sp +\fBWarning:\fP Please note that partial repository checks (i.e. running it with +\fB\-\-max\-duration\fP) can only perform non\-cryptographic checksum checks on the +segment files. A full repository check (i.e. without \fB\-\-max\-duration\fP) can +also do a repository index check. Enabling partial repository checks excepts +archive checks for the same reason. Therefore partial checks may be useful with +very large repositories only where a full check would take too long. +.sp +The \fB\-\-verify\-data\fP option will perform a full integrity verification (as +opposed to checking the CRC32 of the segment) of data, which means reading the +data from the repository, decrypting and decompressing it. It is a complete +cryptographic verification and hence very time consuming, but will detect any +accidental and malicious corruption. Tamper\-resistance is only guaranteed for +encrypted repositories against attackers without access to the keys. You can +not use \fB\-\-verify\-data\fP with \fB\-\-repository\-only\fP\&. +.SS About repair mode +.sp +The check command is a readonly task by default. If any corruption is found, +Borg will report the issue and proceed with checking. To actually repair the +issues found, pass \fB\-\-repair\fP\&. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fB\-\-repair\fP is a \fBPOTENTIALLY DANGEROUS FEATURE\fP and might lead to data +loss! This does not just include data that was previously lost anyway, but +might include more data for kinds of corruption it is not capable of +dealing with. \fBBE VERY CAREFUL!\fP +.UNINDENT +.UNINDENT .sp Pursuant to the previous warning it is also highly recommended to test the -reliability of the hardware running this software with stress testing software -such as memory testers. Unreliable hardware can also lead to data loss especially -when this command is run in repair mode. +reliability of the hardware running Borg with stress testing software. This +especially includes storage and memory testers. Unreliable hardware might lead +to additional data loss. .sp -First, the underlying repository data files are checked: +It is highly recommended to create a backup of your repository before running +in repair mode (i.e. running it with \fB\-\-repair\fP). +.sp +Repair mode will attempt to fix any corruptions found. Fixing corruptions does +not mean recovering lost data: Borg can not magically restore data lost due to +e.g. a hardware failure. Repairing a repository means sacrificing some data +for the sake of the repository as a whole and the remaining data. Hence it is, +by definition, a potentially lossy task. +.sp +In practice, repair mode hooks into both the repository and archive checks: .INDENT 0.0 -.IP \(bu 2 -For all segments, the segment magic header is checked. -.IP \(bu 2 -For all objects stored in the segments, all metadata (e.g. CRC and size) and -all data is read. The read data is checked by size and CRC. Bit rot and other -types of accidental damage can be detected this way. -.IP \(bu 2 -In repair mode, if an integrity error is detected in a segment, try to recover -as many objects from the segment as possible. -.IP \(bu 2 -In repair mode, make sure that the index is consistent with the data stored in -the segments. -.IP \(bu 2 -If checking a remote repo via \fBssh:\fP, the repo check is executed on the server -without causing significant network traffic. -.IP \(bu 2 -The repository check can be skipped using the \fB\-\-archives\-only\fP option. -.IP \(bu 2 -A repository check can be time consuming. Partial checks are possible with the -\fB\-\-max\-duration\fP option. +.IP 1. 3 +When checking the repository\(aqs consistency, repair mode will try to recover +as many objects from segments with integrity errors as possible, and ensure +that the index is consistent with the data stored in the segments. +.IP 2. 3 +When checking the consistency and correctness of archives, repair mode might +remove whole archives from the manifest if their archive metadata chunk is +corrupt or lost. On a chunk level (i.e. the contents of files), repair mode +will replace corrupt or lost chunks with a same\-size replacement chunk of +zeroes. If a previously zeroed chunk reappears, repair mode will restore +this lost chunk using the new chunk. Lastly, repair mode will also delete +orphaned chunks (e.g. caused by read errors while creating the archive). .UNINDENT .sp -Second, the consistency and correctness of the archive metadata is verified: -.INDENT 0.0 -.IP \(bu 2 -Is the repo manifest present? If not, it is rebuilt from archive metadata -chunks (this requires reading and decrypting of all metadata and data). -.IP \(bu 2 -Check if archive metadata chunk is present; if not, remove archive from manifest. -.IP \(bu 2 -For all files (items) in the archive, for all chunks referenced by these -files, check if chunk is present. In repair mode, if a chunk is not present, -replace it with a same\-size replacement chunk of zeroes. If a previously lost -chunk reappears (e.g. via a later backup), in repair mode the all\-zero replacement -chunk will be replaced by the correct chunk. This requires reading of archive and -file metadata, but not data. -.IP \(bu 2 -In repair mode, when all the archives were checked, orphaned chunks are deleted -from the repo. One cause of orphaned chunks are input file related errors (like -read errors) in the archive creation process. -.IP \(bu 2 -In verify\-data mode, a complete cryptographic verification of the archive data -integrity is performed. This conflicts with \fB\-\-repository\-only\fP as this mode -only makes sense if the archive checks are enabled. The full details of this mode -are documented below. -.IP \(bu 2 -If checking a remote repo via \fBssh:\fP, the archive check is executed on the -client machine because it requires decryption, and this is always done client\-side -as key access is needed. -.IP \(bu 2 -The archive checks can be time consuming; they can be skipped using the -\fB\-\-repository\-only\fP option. -.UNINDENT +Most steps taken by repair mode have a one\-time effect on the repository, like +removing a lost archive from the repository. However, replacing a corrupt or +lost chunk with an all\-zero replacement will have an ongoing effect on the +repository: When attempting to extract a file referencing an all\-zero chunk, +the \fBextract\fP command will distinctly warn about it. The FUSE filesystem +created by the \fBmount\fP command will reject reading such a \(dqzero\-patched\(dq +file unless a special mount option is given. .sp -The \fB\-\-max\-duration\fP option can be used to split a long\-running repository check -into multiple partial checks. After the given number of seconds the check is -interrupted. The next partial check will continue where the previous one stopped, -until the complete repository has been checked. Example: Assuming a complete check took 7 -hours, then running a daily check with \-\-max\-duration=3600 (1 hour) resulted in one -completed check per week. -.sp -Attention: A partial \-\-repository\-only check can only do way less checking than a full -\-\-repository\-only check: only the non\-cryptographic checksum checks on segment file -entries are done, while a full \-\-repository\-only check would also do a repo index check. -A partial check cannot be combined with the \fB\-\-repair\fP option. Partial checks -may therefore be useful only with very large repositories where a full check would take -too long. -Doing a full repository check aborts a partial check; the next partial check will restart -from the beginning. -.sp -The \fB\-\-verify\-data\fP option will perform a full integrity verification (as opposed to -checking the CRC32 of the segment) of data, which means reading the data from the -repository, decrypting and decompressing it. This is a cryptographic verification, -which will detect (accidental) corruption. For encrypted repositories it is -tamper\-resistant as well, unless the attacker has access to the keys. It is also very -slow. +As mentioned earlier, Borg might be able to \(dqheal\(dq a \(dqzero\-patched\(dq file in +repair mode, if all its previously lost chunks reappear (e.g. via a later +backup). This is achieved by Borg not only keeping track of the all\-zero +replacement chunks, but also by keeping metadata about the lost chunks. In +repair mode Borg will check whether a previously lost chunk reappeared and will +replace the all\-zero replacement chunk by the reappeared chunk. If all lost +chunks of a \(dqzero\-patched\(dq file reappear, this effectively \(dqheals\(dq the file. +Consequently, if lost chunks were repaired earlier, it is advised to run +\fB\-\-repair\fP a second time after creating some new backups. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index cc65bce9e..dc37457fb 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMMON" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-COMMON" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands .SH SYNOPSIS diff --git a/docs/man/borg-compact.1 b/docs/man/borg-compact.1 index 595030b34..1aa765eac 100644 --- a/docs/man/borg-compact.1 +++ b/docs/man/borg-compact.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMPACT" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-COMPACT" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-compact \- compact segment files in the repository .SH SYNOPSIS diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index e4adfdca3..bc65a9886 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMPRESSION" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-COMPRESSION" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression .SH DESCRIPTION @@ -56,7 +56,6 @@ Use lz4 compression. Very high speed, very low compression. (default) Use zstd (\(dqzstandard\(dq) compression, a modern wide\-range algorithm. If you do not explicitly give the compression level L (ranging from 1 to 22), it will use level 3. -Archives compressed with zstd are not compatible with borg < 1.1.4. .TP .B zlib[,L] Use zlib (\(dqgz\(dq) compression. Medium speed, medium compression. diff --git a/docs/man/borg-config.1 b/docs/man/borg-config.1 index a521084c7..5abc675d6 100644 --- a/docs/man/borg-config.1 +++ b/docs/man/borg-config.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CONFIG" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-CONFIG" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-config \- get, set, and delete values in a repository or cache config file .SH SYNOPSIS diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 669a4cf47..d0f497569 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CREATE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-CREATE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-create \- Create new archive .SH SYNOPSIS @@ -299,8 +299,7 @@ $ cd .. $ fusermount \-u sshfs\-mount # Make a big effort in fine granular deduplication (big chunk management -# overhead, needs a lot of RAM and disk space, see formula in internals -# docs \- same parameters as borg < 1.0): +# overhead, needs a lot of RAM and disk space, see formula in internals docs): $ borg create \-\-chunker\-params buzhash,10,23,16,4095 small /smallstuff # Backup a raw device (must not be active/in use/mounted at that time) diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 5537168cb..11bfe4bb0 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-DELETE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-DELETE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-delete \- Delete archives .SH SYNOPSIS @@ -110,7 +110,7 @@ consider archives newer than (now \- TIMESPAN), e.g. 7d or 12m. .nf .ft C # delete a single backup archive: -$ borg delete Monday +$ borg delete \-a Monday # actually free disk space: $ borg compact diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 8c73f281d..0f84b9966 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-DIFF" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-DIFF" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives .SH SYNOPSIS diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 792c812e0..a82f16207 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-EXPORT-TAR" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-EXPORT-TAR" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball .SH SYNOPSIS diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index d5d734aa9..89989def5 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-EXTRACT" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-EXTRACT" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents .SH SYNOPSIS diff --git a/docs/man/borg-import-tar.1 b/docs/man/borg-import-tar.1 index 7714750cc..d84c53a9b 100644 --- a/docs/man/borg-import-tar.1 +++ b/docs/man/borg-import-tar.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-IMPORT-TAR" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-IMPORT-TAR" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-import-tar \- Create a backup archive from a tarball .SH SYNOPSIS diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 86940f570..4962ddd2d 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-INFO" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-INFO" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used .SH SYNOPSIS diff --git a/docs/man/borg-key-change-location.1 b/docs/man/borg-key-change-location.1 index ead426667..292200f3a 100644 --- a/docs/man/borg-key-change-location.1 +++ b/docs/man/borg-key-change-location.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-CHANGE-LOCATION" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-KEY-CHANGE-LOCATION" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-key-change-location \- Change repository key location .SH SYNOPSIS diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 74ebf60dd..7f2468615 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-CHANGE-PASSPHRASE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-KEY-CHANGE-PASSPHRASE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase .SH SYNOPSIS diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 1ed16caab..c4916175b 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-EXPORT" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-KEY-EXPORT" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup .SH SYNOPSIS diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 97c8c8130..7aca05fc5 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-IMPORT" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-KEY-IMPORT" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup .SH SYNOPSIS diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index bc984eefe..18287bee3 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-KEY" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository .SH SYNOPSIS diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index b54c78a50..d6868c8ad 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-LIST" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-LIST" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-list \- List archive contents .SH SYNOPSIS @@ -106,11 +106,11 @@ drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.text \&... -$ borg list archiveA \-\-pattern \(aqre:\e.ext$\(aq +$ borg list archiveA \-\-pattern \(aq+ re:\e.ext$\(aq \-\-pattern \(aq\- re:^.*$\(aq \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext \&... -$ borg list archiveA \-\-pattern \(aqre:.ext$\(aq +$ borg list archiveA \-\-pattern \(aq+ re:.ext$\(aq \-\-pattern \(aq\- re:^.*$\(aq \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.text \&... diff --git a/docs/man/borg-match-archives.1 b/docs/man/borg-match-archives.1 index 7e6344529..3dfb70b9d 100644 --- a/docs/man/borg-match-archives.1 +++ b/docs/man/borg-match-archives.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-MATCH-ARCHIVES" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-MATCH-ARCHIVES" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-match-archives \- Details regarding match-archives .SH DESCRIPTION diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 828c4e5d7..9e06af762 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-MOUNT" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-MOUNT" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem .SH SYNOPSIS diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index a4918695b..545bed825 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PATTERNS" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-PATTERNS" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns .SH DESCRIPTION diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 8810acb99..175e27e7a 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PLACEHOLDERS" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-PLACEHOLDERS" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders .SH DESCRIPTION diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 01617e346..8e570016a 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PRUNE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-PRUNE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules .SH SYNOPSIS diff --git a/docs/man/borg-rcompress.1 b/docs/man/borg-rcompress.1 index 14b796947..ea0e2adec 100644 --- a/docs/man/borg-rcompress.1 +++ b/docs/man/borg-rcompress.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RCOMPRESS" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RCOMPRESS" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-rcompress \- Repository (re-)compression .SH SYNOPSIS diff --git a/docs/man/borg-rcreate.1 b/docs/man/borg-rcreate.1 index 350fda922..987d96382 100644 --- a/docs/man/borg-rcreate.1 +++ b/docs/man/borg-rcreate.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RCREATE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RCREATE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-rcreate \- Create a new, empty repository .SH SYNOPSIS @@ -211,9 +211,12 @@ case of malicious activity in the repository. .sp If you do \fBnot\fP want to encrypt the contents of your backups, but still want to detect malicious tampering use an \fIauthenticated\fP mode. It\(aqs like \fIrepokey\fP minus encryption. +To normally work with \fBauthenticated\fP repos, you will need the passphrase, but +there is an emergency workaround, see \fBBORG_WORKAROUNDS=authenticated_no_key\fP docs. .SS Creating a related repository .sp -A related repository uses same secret key material as the other/original repository. +You can use \fBborg rcreate \-\-other\-repo ORIG_REPO ...\fP to create a related repository +that uses the same secret key material as the given other/original repository. .sp By default, only the ID key and chunker secret will be the same (these are important for deduplication) and the AE crypto keys will be newly generated random keys. diff --git a/docs/man/borg-rdelete.1 b/docs/man/borg-rdelete.1 index 87f6e4ca3..eca75a619 100644 --- a/docs/man/borg-rdelete.1 +++ b/docs/man/borg-rdelete.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RDELETE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RDELETE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-rdelete \- Delete a repository .SH SYNOPSIS diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 3a8174d17..707b2c307 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RECREATE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RECREATE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives .SH SYNOPSIS diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 44bd79f72..1ce495330 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RENAME" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RENAME" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive .SH SYNOPSIS diff --git a/docs/man/borg-rinfo.1 b/docs/man/borg-rinfo.1 index a87a0ab4f..3d6e8e01a 100644 --- a/docs/man/borg-rinfo.1 +++ b/docs/man/borg-rinfo.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RINFO" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RINFO" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-rinfo \- Show repository infos .SH SYNOPSIS diff --git a/docs/man/borg-rlist.1 b/docs/man/borg-rlist.1 index e388e9af2..1c56abfa2 100644 --- a/docs/man/borg-rlist.1 +++ b/docs/man/borg-rlist.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RLIST" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-RLIST" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-rlist \- List the archives contained in a repository .SH SYNOPSIS @@ -117,7 +117,7 @@ ArchiveBar # Strings are left\-aligned, numbers are right\-aligned. # Note: time columns except \(ga\(gaisomtime\(ga\(ga, \(ga\(gaisoctime\(ga\(ga and \(ga\(gaisoatime\(ga\(ga cannot be padded. $ borg rlist \-\-format \(aq{archive:36} {time} [{id}]{NL}\(aq /path/to/repo -ArchiveFoo Thu, 2021\-12\-09 10:22:28 [0b8e9a312bef3f2f6e2d0fc110c196827786c15eba0188738e81697a7fa3b274] +ArchiveFoo Thu, 2021\-12\-09 10:22:28 [0b8e9...3b274] \&... .ft P .fi @@ -153,6 +153,8 @@ comment: archive comment .IP \(bu 2 id: internal ID of the archive .IP \(bu 2 +tam: TAM authentication state of this archive +.IP \(bu 2 start: time (start) of creation of the archive .IP \(bu 2 time: alias of \(dqstart\(dq diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 10c8db5b0..1792ef5d6 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-SERVE" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-SERVE" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. .SH SYNOPSIS diff --git a/docs/man/borg-transfer.1 b/docs/man/borg-transfer.1 index 0c633d4f8..73fb58ec6 100644 --- a/docs/man/borg-transfer.1 +++ b/docs/man/borg-transfer.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-TRANSFER" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-TRANSFER" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-transfer \- archives transfer from other repository, optionally upgrade data format .SH SYNOPSIS diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 04486913a..5d04e3f2c 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-UMOUNT" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-UMOUNT" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem .SH SYNOPSIS diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 75a0c9d02..52000a0aa 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-WITH-LOCK" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG-WITH-LOCK" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held .SH SYNOPSIS diff --git a/docs/man/borg.1 b/docs/man/borg.1 index 44c055b29..9849fb6ae 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG" 1 "2023-06-11" "" "borg backup tool" +.TH "BORG" 1 "2023-09-14" "" "borg backup tool" .SH NAME borg \- deduplicating and encrypting backup tool .SH SYNOPSIS @@ -493,6 +493,38 @@ write to disk behaviour (not continuously streaming to disk). Retry opening a file without O_NOATIME if opening a file with O_NOATIME caused EROFS. You will need this to make archives from volume shadow copies in WSL1 (Windows Subsystem for Linux 1). +.TP +.B authenticated_no_key +Work around a lost passphrase or key for an \fBauthenticated\fP mode repository +(these are only authenticated, but not encrypted). +If the key is missing in the repository config, add \fBkey = anything\fP there. +.sp +This workaround is \fBonly\fP for emergencies and \fBonly\fP to extract data +from an affected repository (read\-only access): +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +BORG_WORKAROUNDS=authenticated_no_key borg extract repo::archive +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +After you have extracted all data you need, you MUST delete the repository: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +BORG_WORKAROUNDS=authenticated_no_key borg delete repo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Now you can init a fresh repo. Make sure you do not use the workaround any more. .UNINDENT .UNINDENT .TP @@ -646,7 +678,7 @@ At least three directory levels with short names Typically, file sizes up to a few hundred MB. Large repositories may require large files (>2 GB). .IP \(bu 2 -Up to 1000 files per directory (10000 for repositories initialized with Borg 1.0) +Up to 1000 files per directory. .IP \(bu 2 rename(2) / MoveFile(Ex) should work as specified, i.e. on the same file system it should be a move (not a copy) operation, and in case of a directory @@ -842,7 +874,7 @@ Yes [1] T} _ T{ -Mac OS X +macOS T} T{ Yes T} T{ diff --git a/docs/man/borgfs.1 b/docs/man/borgfs.1 index 3db9be0a0..aae8cf598 100644 --- a/docs/man/borgfs.1 +++ b/docs/man/borgfs.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORGFS" 1 "2023-06-11" "" "borg backup tool" +.TH "BORGFS" 1 "2023-09-14" "" "borg backup tool" .SH NAME borgfs \- Mount archive or an entire repository as a FUSE filesystem .SH SYNOPSIS diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index 12605e09d..4dcc593d6 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -85,75 +85,115 @@ borg check Description ~~~~~~~~~~~ -The check command verifies the consistency of a repository and the corresponding archives. +The check command verifies the consistency of a repository and its archives. +It consists of two major steps: -check --repair is a potentially dangerous function and might lead to data loss -(for kinds of corruption it is not capable of dealing with). BE VERY CAREFUL! +1. Checking the consistency of the repository itself. This includes checking + the segment magic headers, and both the metadata and data of all objects in + the segments. The read data is checked by size and CRC. Bit rot and other + types of accidental damage can be detected this way. Running the repository + check can be split into multiple partial checks using ``--max-duration``. + When checking a remote repository, please note that the checks run on the + server and do not cause significant network traffic. + +2. Checking consistency and correctness of the archive metadata and optionally + archive data (requires ``--verify-data``). This includes ensuring that the + repository manifest exists, the archive metadata chunk is present, and that + all chunks referencing files (items) in the archive exist. This requires + reading archive and file metadata, but not data. To cryptographically verify + the file (content) data integrity pass ``--verify-data``, but keep in mind + that this requires reading all data and is hence very time consuming. When + checking archives of a remote repository, archive checks run on the client + machine because they require decrypting data and therefore the encryption + key. + +Both steps can also be run independently. Pass ``--repository-only`` to run the +repository checks only, or pass ``--archives-only`` to run the archive checks +only. + +The ``--max-duration`` option can be used to split a long-running repository +check into multiple partial checks. After the given number of seconds the check +is interrupted. The next partial check will continue where the previous one +stopped, until the full repository has been checked. Assuming a complete check +would take 7 hours, then running a daily check with ``--max-duration=3600`` +(1 hour) would result in one full repository check per week. Doing a full +repository check aborts any previous partial check; the next partial check will +restart from the beginning. With partial repository checks you can run neither +archive checks, nor enable repair mode. Consequently, if you want to use +``--max-duration`` you must also pass ``--repository-only``, and must not pass +``--archives-only``, nor ``--repair``. + +**Warning:** Please note that partial repository checks (i.e. running it with +``--max-duration``) can only perform non-cryptographic checksum checks on the +segment files. A full repository check (i.e. without ``--max-duration``) can +also do a repository index check. Enabling partial repository checks excepts +archive checks for the same reason. Therefore partial checks may be useful with +very large repositories only where a full check would take too long. + +The ``--verify-data`` option will perform a full integrity verification (as +opposed to checking the CRC32 of the segment) of data, which means reading the +data from the repository, decrypting and decompressing it. It is a complete +cryptographic verification and hence very time consuming, but will detect any +accidental and malicious corruption. Tamper-resistance is only guaranteed for +encrypted repositories against attackers without access to the keys. You can +not use ``--verify-data`` with ``--repository-only``. + +About repair mode ++++++++++++++++++ + +The check command is a readonly task by default. If any corruption is found, +Borg will report the issue and proceed with checking. To actually repair the +issues found, pass ``--repair``. + +.. note:: + + ``--repair`` is a **POTENTIALLY DANGEROUS FEATURE** and might lead to data + loss! This does not just include data that was previously lost anyway, but + might include more data for kinds of corruption it is not capable of + dealing with. **BE VERY CAREFUL!** Pursuant to the previous warning it is also highly recommended to test the -reliability of the hardware running this software with stress testing software -such as memory testers. Unreliable hardware can also lead to data loss especially -when this command is run in repair mode. +reliability of the hardware running Borg with stress testing software. This +especially includes storage and memory testers. Unreliable hardware might lead +to additional data loss. -First, the underlying repository data files are checked: +It is highly recommended to create a backup of your repository before running +in repair mode (i.e. running it with ``--repair``). -- For all segments, the segment magic header is checked. -- For all objects stored in the segments, all metadata (e.g. CRC and size) and - all data is read. The read data is checked by size and CRC. Bit rot and other - types of accidental damage can be detected this way. -- In repair mode, if an integrity error is detected in a segment, try to recover - as many objects from the segment as possible. -- In repair mode, make sure that the index is consistent with the data stored in - the segments. -- If checking a remote repo via ``ssh:``, the repo check is executed on the server - without causing significant network traffic. -- The repository check can be skipped using the ``--archives-only`` option. -- A repository check can be time consuming. Partial checks are possible with the - ``--max-duration`` option. +Repair mode will attempt to fix any corruptions found. Fixing corruptions does +not mean recovering lost data: Borg can not magically restore data lost due to +e.g. a hardware failure. Repairing a repository means sacrificing some data +for the sake of the repository as a whole and the remaining data. Hence it is, +by definition, a potentially lossy task. -Second, the consistency and correctness of the archive metadata is verified: +In practice, repair mode hooks into both the repository and archive checks: -- Is the repo manifest present? If not, it is rebuilt from archive metadata - chunks (this requires reading and decrypting of all metadata and data). -- Check if archive metadata chunk is present; if not, remove archive from manifest. -- For all files (items) in the archive, for all chunks referenced by these - files, check if chunk is present. In repair mode, if a chunk is not present, - replace it with a same-size replacement chunk of zeroes. If a previously lost - chunk reappears (e.g. via a later backup), in repair mode the all-zero replacement - chunk will be replaced by the correct chunk. This requires reading of archive and - file metadata, but not data. -- In repair mode, when all the archives were checked, orphaned chunks are deleted - from the repo. One cause of orphaned chunks are input file related errors (like - read errors) in the archive creation process. -- In verify-data mode, a complete cryptographic verification of the archive data - integrity is performed. This conflicts with ``--repository-only`` as this mode - only makes sense if the archive checks are enabled. The full details of this mode - are documented below. -- If checking a remote repo via ``ssh:``, the archive check is executed on the - client machine because it requires decryption, and this is always done client-side - as key access is needed. -- The archive checks can be time consuming; they can be skipped using the - ``--repository-only`` option. +1. When checking the repository's consistency, repair mode will try to recover + as many objects from segments with integrity errors as possible, and ensure + that the index is consistent with the data stored in the segments. -The ``--max-duration`` option can be used to split a long-running repository check -into multiple partial checks. After the given number of seconds the check is -interrupted. The next partial check will continue where the previous one stopped, -until the complete repository has been checked. Example: Assuming a complete check took 7 -hours, then running a daily check with --max-duration=3600 (1 hour) resulted in one -completed check per week. +2. When checking the consistency and correctness of archives, repair mode might + remove whole archives from the manifest if their archive metadata chunk is + corrupt or lost. On a chunk level (i.e. the contents of files), repair mode + will replace corrupt or lost chunks with a same-size replacement chunk of + zeroes. If a previously zeroed chunk reappears, repair mode will restore + this lost chunk using the new chunk. Lastly, repair mode will also delete + orphaned chunks (e.g. caused by read errors while creating the archive). -Attention: A partial --repository-only check can only do way less checking than a full ---repository-only check: only the non-cryptographic checksum checks on segment file -entries are done, while a full --repository-only check would also do a repo index check. -A partial check cannot be combined with the ``--repair`` option. Partial checks -may therefore be useful only with very large repositories where a full check would take -too long. -Doing a full repository check aborts a partial check; the next partial check will restart -from the beginning. +Most steps taken by repair mode have a one-time effect on the repository, like +removing a lost archive from the repository. However, replacing a corrupt or +lost chunk with an all-zero replacement will have an ongoing effect on the +repository: When attempting to extract a file referencing an all-zero chunk, +the ``extract`` command will distinctly warn about it. The FUSE filesystem +created by the ``mount`` command will reject reading such a "zero-patched" +file unless a special mount option is given. -The ``--verify-data`` option will perform a full integrity verification (as opposed to -checking the CRC32 of the segment) of data, which means reading the data from the -repository, decrypting and decompressing it. This is a cryptographic verification, -which will detect (accidental) corruption. For encrypted repositories it is -tamper-resistant as well, unless the attacker has access to the keys. It is also very -slow. \ No newline at end of file +As mentioned earlier, Borg might be able to "heal" a "zero-patched" file in +repair mode, if all its previously lost chunks reappear (e.g. via a later +backup). This is achieved by Borg not only keeping track of the all-zero +replacement chunks, but also by keeping metadata about the lost chunks. In +repair mode Borg will check whether a previously lost chunk reappeared and will +replace the all-zero replacement chunk by the reappeared chunk. If all lost +chunks of a "zero-patched" file reappear, this effectively "heals" the file. +Consequently, if lost chunks were repaired earlier, it is advised to run +``--repair`` a second time after creating some new backups. \ No newline at end of file diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 1fee15d4e..4833db2dc 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -385,7 +385,6 @@ zstd[,L] Use zstd ("zstandard") compression, a modern wide-range algorithm. If you do not explicitly give the compression level L (ranging from 1 to 22), it will use level 3. - Archives compressed with zstd are not compatible with borg < 1.1.4. zlib[,L] Use zlib ("gz") compression. Medium speed, medium compression. diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index db947392a..c95788269 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -139,6 +139,12 @@ from different machines) in one shared repository, use one prune call per data set that matches only the respective archives using the --match-archives (-a) option. +The ``--keep-within`` option takes an argument of the form "", +where char is "H", "d", "w", "m", "y". For example, ``--keep-within 2d`` means +to keep all archives that were created within the past 48 hours. +"1m" is taken to mean "31d". The archives kept with this option do not +count towards the totals specified by any other options. + A good procedure is to thin out more and more the older your backups get. As an example, ``--keep-daily 7`` means to keep the latest backup on each day, up to 7 most recent days with backups (days without backups do not count). @@ -152,13 +158,6 @@ minutely, hourly, daily, weekly, monthly, or yearly rules was not otherwise able meet its retention target. This enables the first chronological archive to continue aging until it is replaced by a newer archive that meets the retention criteria. -The ``--keep-within`` option takes an argument of the form "", -where char is "H", "d", "w", "m", "y". For example, ``--keep-within 2d`` means -to keep all archives that were created within the past 48 hours. -"1m" is taken to mean "31d". This option is applied before the secondly option -and like the other options any archives kept by this option do not count towards -the later rules. - The ``--keep-last N`` option is doing the same as ``--keep-secondly N`` (and it will keep the last N archives under the assumption that you do not create more than one backup archive in the same second). @@ -170,4 +169,4 @@ Please note that the "All archives" stats refer to the state after pruning. You can influence how the ``--list`` output is formatted by using the ``--short`` option (less wide output) or by giving a custom format using ``--format`` (see -the ``borg rlist`` description for more details about the format string). +the ``borg rlist`` description for more details about the format string). \ No newline at end of file diff --git a/docs/usage/rcreate.rst.inc b/docs/usage/rcreate.rst.inc index c6ef03b23..8cdf8e31d 100644 --- a/docs/usage/rcreate.rst.inc +++ b/docs/usage/rcreate.rst.inc @@ -157,11 +157,14 @@ case of malicious activity in the repository. If you do **not** want to encrypt the contents of your backups, but still want to detect malicious tampering use an `authenticated` mode. It's like `repokey` minus encryption. +To normally work with ``authenticated`` repos, you will need the passphrase, but +there is an emergency workaround, see ``BORG_WORKAROUNDS=authenticated_no_key`` docs. Creating a related repository +++++++++++++++++++++++++++++ -A related repository uses same secret key material as the other/original repository. +You can use ``borg rcreate --other-repo ORIG_REPO ...`` to create a related repository +that uses the same secret key material as the given other/original repository. By default, only the ID key and chunker secret will be the same (these are important for deduplication) and the AE crypto keys will be newly generated random keys. diff --git a/docs/usage/rlist.rst.inc b/docs/usage/rlist.rst.inc index f4fc67ab5..c05ef4a9b 100644 --- a/docs/usage/rlist.rst.inc +++ b/docs/usage/rlist.rst.inc @@ -104,7 +104,7 @@ Examples: # Strings are left-aligned, numbers are right-aligned. # Note: time columns except ``isomtime``, ``isoctime`` and ``isoatime`` cannot be padded. $ borg rlist --format '{archive:36} {time} [{id}]{NL}' /path/to/repo - ArchiveFoo Thu, 2021-12-09 10:22:28 [0b8e9a312bef3f2f6e2d0fc110c196827786c15eba0188738e81697a7fa3b274] + ArchiveFoo Thu, 2021-12-09 10:22:28 [0b8e9...3b274] ... The following keys are always available: @@ -124,6 +124,7 @@ Keys available only when listing archives in a repository: - name: alias of "archive" - comment: archive comment - id: internal ID of the archive +- tam: TAM authentication state of this archive - start: time (start) of creation of the archive - time: alias of "start"