diff --git a/docs/changes.rst b/docs/changes.rst index 4c2430f72..71f05d471 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,6 +5,72 @@ Important notes This section provides information about security and corruption issues. +.. _archives_tam_vuln: + +Pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811) +---------------------------------------------------------- + +A flaw in the cryptographic authentication scheme in Borg allowed an attacker to +fake archives and potentially indirectly cause backup data loss in the repository. + +The attack requires an attacker to be able to + +1. insert files (with no additional headers) into backups +2. gain write access to the repository + +This vulnerability does not disclose plaintext to the attacker, nor does it +affect the authenticity of existing archives. + +Creating plausible fake archives may be feasible for empty or small archives, +but is unlikely for large archives. + +The fix enforces checking the TAM authentication tag of archives at critical +places. Borg now considers archives without TAM as garbage or an attack. + +We are not aware of others having discovered, disclosed or exploited this vulnerability. + +Below, if we speak of borg 1.2.5, we mean a borg version >= 1.2.5 **or** a +borg version that has the relevant security patches for this vulnerability applied +(could be also an older version in that case). + +Steps you must take to upgrade a repository: + +1. Upgrade all clients using this repository to borg 1.2.5. + Note: it is not required to upgrade a server, except if the server-side borg + is also used as a client (and not just for "borg serve"). + + Do **not** run ``borg check`` with borg 1.2.5 before completing the upgrade steps. + +2. Run ``borg info --debug 2>&1 | grep TAM | grep -i manifest``. + a) If you get "TAM-verified manifest", continue with 3. + b) If you get "Manifest TAM not found and not required", run + ``borg upgrade --tam --force `` *on every client*. + +3. Run ``borg list --format='{name} {time} tam:{tam}{NL}' ``. + "tam:verified" means that the archive has a valid TAM authentication. + "tam:none" is expected as output for archives created by borg <1.0.9. + "tam:none" could also come from archives created by an attacker. + You should verify that "tam:none" archives are authentic and not malicious + (== have good content, have correct timestamp, can be extracted successfully). + In case you find crappy/malicious archives, you must delete them before proceeding. + In low-risk, trusted environments, you may decide on your own risk to skip step 3 + and just trust in everything being OK. + +4. If there are no tam:non archives left at this point, you can skip this step. + Run ``borg upgrade --archives-tam ``. + This will make sure all archives are TAM authenticated (an archive TAM will be added + for all archives still missing one). + ``borg check`` would consider TAM-less archives as garbage or a potential attack. + Optionally run the same command as in step 3 to see that all archives now are "tam:verified". + + +Vulnerability time line: + +* 2023-06-13: Vulnerability discovered during code review by Thomas Waldmann +* 2023-06-13...: Work on fixing the issue, upgrade procedure, docs. +* 2023-06-30: CVE was assigned via Github CNA +* 2023-07-xx: Released fixed version 1.2.5 + .. _hashindex_set_bug: Pre-1.1.11 potential index corruption / data loss issue @@ -242,6 +308,8 @@ Some things can be recommended for the upgrade process from borg 1.1.x take significant time, but after that it will be fast) - for more details see below. - check the compatibility notes (see below) and adapt your scripts, if needed. +- borg 1.2.5 has a security fix for the pre-1.2.5 archives spoofing vulnerability + (CVE-2023-36811), see details and necessary upgrade procedure described above. - if you run into any issues, please check the github issue tracker before posting new issues there or elsewhere. @@ -299,14 +367,16 @@ Compatibility notes: Change Log ========== -Version 1.2.5 (not released yet) --------------------------------- +Version 1.2.5 (2023-08-30) +-------------------------- For upgrade and compatibility hints, please also read the section "Upgrade Notes" above. Fixes: +- Security: fix pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811), + see details and necessary upgrade procedure described above. - create: do not try to read parent dir of recursion root, #7746 - extract: fix false warning about pattern never matching, #4110 - diff: remove surrogates before output, #7535 diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index f39610d1d..9f504102b 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK-CRUD" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-BENCHMARK-CRUD" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. .SH SYNOPSIS diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 4859492d9..a3aacd355 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-BENCHMARK" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command .SH SYNOPSIS diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 25f5a62cd..49d3ee59b 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BREAK-LOCK" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-BREAK-LOCK" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. .SH SYNOPSIS diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 7db3267e0..85125bb1a 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CHECK" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-CHECK" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency .SH SYNOPSIS diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index fe99b1d42..b5a761962 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMMON" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-COMMON" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands .SH SYNOPSIS diff --git a/docs/man/borg-compact.1 b/docs/man/borg-compact.1 index bf05ab41a..e7cf87f79 100644 --- a/docs/man/borg-compact.1 +++ b/docs/man/borg-compact.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMPACT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-COMPACT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-compact \- compact segment files in the repository .SH SYNOPSIS diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 47d9ea897..e808ac6d6 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMPRESSION" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-COMPRESSION" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression .SH DESCRIPTION diff --git a/docs/man/borg-config.1 b/docs/man/borg-config.1 index 750b63715..527406686 100644 --- a/docs/man/borg-config.1 +++ b/docs/man/borg-config.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CONFIG" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-CONFIG" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-config \- get, set, and delete values in a repository or cache config file .SH SYNOPSIS diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index dab725d2b..2c097ec17 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CREATE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-CREATE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-create \- Create new archive .SH SYNOPSIS diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 87869dc34..4d2808a5d 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-DELETE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-DELETE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives .SH SYNOPSIS diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 84c2a5517..cedbbade2 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-DIFF" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-DIFF" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives .SH SYNOPSIS diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index e8ddee3ed..03a559818 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-EXPORT-TAR" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-EXPORT-TAR" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball .SH SYNOPSIS diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 55c47f50e..365e94e5a 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-EXTRACT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-EXTRACT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents .SH SYNOPSIS diff --git a/docs/man/borg-import-tar.1 b/docs/man/borg-import-tar.1 index 14af072d4..77554154b 100644 --- a/docs/man/borg-import-tar.1 +++ b/docs/man/borg-import-tar.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-IMPORT-TAR" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-IMPORT-TAR" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-import-tar \- Create a backup archive from a tarball .SH SYNOPSIS diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index d6c886c3e..c1b209f21 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-INFO" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-INFO" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used .SH SYNOPSIS diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index ccf0b06f4..334468245 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-INIT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-INIT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository .SH SYNOPSIS diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 50d0e7996..c55be59b3 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-CHANGE-PASSPHRASE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-KEY-CHANGE-PASSPHRASE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase .SH SYNOPSIS diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 03528d8c3..aa9950502 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-EXPORT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-KEY-EXPORT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup .SH SYNOPSIS diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 8a2742d72..135310813 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-IMPORT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-KEY-IMPORT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup .SH SYNOPSIS diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 8eda9b4d7..a67ca07e6 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-MIGRATE-TO-REPOKEY" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-KEY-MIGRATE-TO-REPOKEY" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey .SH SYNOPSIS diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 2dac99a44..b3c3a9070 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-KEY" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository .SH SYNOPSIS diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index ebf2582f9..2a42b8ad3 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-LIST" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-LIST" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents .SH SYNOPSIS @@ -217,6 +217,8 @@ bcomment: verbatim archive comment, can contain any character except NUL .IP \(bu 2 id: internal ID of the archive .IP \(bu 2 +tam: TAM authentication state of this archive +.IP \(bu 2 start: time (start) of creation of the archive .IP \(bu 2 time: alias of \(dqstart\(dq diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 27b02579d..d0b706751 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-MOUNT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-MOUNT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem .SH SYNOPSIS diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index a08c0ea18..7eeee6dff 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PATTERNS" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-PATTERNS" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns .SH DESCRIPTION diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 53a87052d..3c7121d4e 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PLACEHOLDERS" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-PLACEHOLDERS" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders .SH DESCRIPTION diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index b7489f733..b38708cb9 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PRUNE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-PRUNE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules .SH SYNOPSIS diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 04d928439..cd6d2b40b 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RECREATE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-RECREATE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives .SH SYNOPSIS diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 627c041c3..22b75cf8b 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-RENAME" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-RENAME" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive .SH SYNOPSIS diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 20f4f3fac..f723db1df 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-SERVE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-SERVE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. .SH SYNOPSIS diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index a10e6b472..7c6e45b59 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-UMOUNT" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-UMOUNT" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem .SH SYNOPSIS diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 9f01dde8e..175ee9494 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-UPGRADE" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-UPGRADE" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version .SH SYNOPSIS @@ -53,6 +53,23 @@ except when noted otherwise in the changelog .UNINDENT .SS Borg 1.x.y upgrades .sp +Archive TAM authentication: +.sp +Use \fBborg upgrade \-\-archives\-tam REPO\fP to add archive TAMs to all +archives that are not TAM authenticated yet. +This is a convenient method to just trust all archives present \- if +an archive does not have TAM authentication yet, a TAM will be added. +Archives created by old borg versions < 1.0.9 do not have TAMs. +Archives created by newer borg version should have TAMs already. +If you have a high risk environment, you should not just run this, +but first verify that the archives are authentic and not malicious +(== have good content, have a good timestamp). +Borg 1.2.5+ needs all archives to be TAM authenticated for safety reasons. +.sp +This upgrade needs to be done once per repository. +.sp +Manifest TAM authentication: +.sp Use \fBborg upgrade \-\-tam REPO\fP to require manifest authentication introduced with Borg 1.0.9 to address security issues. This means that modifying the repository after doing this with a version prior @@ -148,6 +165,9 @@ Enable manifest authentication (in key and cache) (Borg 1.0.9 and later). .TP .B \-\-disable\-tam Disable manifest authentication (in key and cache). +.TP +.B \-\-archives\-tam +add TAM authentication for all archives. .UNINDENT .SH EXAMPLES .INDENT 0.0 diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 5155e70f3..34b6bcbf6 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-WITH-LOCK" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG-WITH-LOCK" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held .SH SYNOPSIS diff --git a/docs/man/borg.1 b/docs/man/borg.1 index a6b405cb8..f09a9fc39 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG" 1 "2023-08-29" "" "borg backup tool" +.TH "BORG" 1 "2023-08-30" "" "borg backup tool" .SH NAME borg \- deduplicating and encrypting backup tool .SH SYNOPSIS diff --git a/docs/man/borgfs.1 b/docs/man/borgfs.1 index a9f5d5fd9..d31d9cd61 100644 --- a/docs/man/borgfs.1 +++ b/docs/man/borgfs.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORGFS" 1 "2023-08-29" "" "borg backup tool" +.TH "BORGFS" 1 "2023-08-30" "" "borg backup tool" .SH NAME borgfs \- Mount archive or an entire repository as a FUSE filesystem .SH SYNOPSIS diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 915daa803..c05b777a3 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -156,6 +156,7 @@ Keys available only when listing archives in a repository: - comment: archive comment interpreted as text (might be missing non-text characters, see bcomment) - bcomment: verbatim archive comment, can contain any character except NUL - id: internal ID of the archive +- tam: TAM authentication state of this archive - start: time (start) of creation of the archive - time: alias of "start" diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index bb7c88d92..de6964385 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -29,6 +29,8 @@ borg upgrade +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+ | | ``--disable-tam`` | Disable manifest authentication (in key and cache). | +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+ + | | ``--archives-tam`` | add TAM authentication for all archives. | + +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+ | .. class:: borg-common-opt-ref | | | | :ref:`common_options` | @@ -54,6 +56,7 @@ borg upgrade --force Force upgrade --tam Enable manifest authentication (in key and cache) (Borg 1.0.9 and later). --disable-tam Disable manifest authentication (in key and cache). + --archives-tam add TAM authentication for all archives. :ref:`common_options` @@ -80,6 +83,23 @@ You do **not** need to run it when: Borg 1.x.y upgrades +++++++++++++++++++ +Archive TAM authentication: + +Use ``borg upgrade --archives-tam REPO`` to add archive TAMs to all +archives that are not TAM authenticated yet. +This is a convenient method to just trust all archives present - if +an archive does not have TAM authentication yet, a TAM will be added. +Archives created by old borg versions < 1.0.9 do not have TAMs. +Archives created by newer borg version should have TAMs already. +If you have a high risk environment, you should not just run this, +but first verify that the archives are authentic and not malicious +(== have good content, have a good timestamp). +Borg 1.2.5+ needs all archives to be TAM authenticated for safety reasons. + +This upgrade needs to be done once per repository. + +Manifest TAM authentication: + Use ``borg upgrade --tam REPO`` to require manifest authentication introduced with Borg 1.0.9 to address security issues. This means that modifying the repository after doing this with a version prior diff --git a/src/borg/archive.py b/src/borg/archive.py index 9c60b611a..4155d2341 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -450,6 +450,7 @@ class Archive: self.name = name # overwritten later with name from archive metadata self.name_in_manifest = name # can differ from .name later (if borg check fixed duplicate archive names) self.comment = None + self.tam_verified = False self.checkpoint_interval = checkpoint_interval self.numeric_ids = numeric_ids self.noatime = noatime @@ -488,7 +489,9 @@ class Archive: def _load_meta(self, id): data = self.key.decrypt(id, self.repository.get(id)) - metadata = ArchiveItem(internal_dict=msgpack.unpackb(data)) + # we do not require TAM for archives, otherwise we can not even borg list a repo with old archives. + archive, self.tam_verified, _ = self.key.unpack_and_verify_archive(data, force_tam_not_required=True) + metadata = ArchiveItem(internal_dict=archive) if metadata.version != 1: raise Exception('Unknown archive metadata version') return metadata @@ -959,7 +962,7 @@ Utilization of max. archive size: {csize_max:.0%} def set_meta(self, key, value): metadata = self._load_meta(self.id) setattr(metadata, key, value) - data = msgpack.packb(metadata.as_dict()) + data = self.key.pack_and_authenticate_metadata(metadata.as_dict(), context=b'archive') new_id = self.key.id_hash(data) self.cache.add_chunk(new_id, data, self.stats) self.manifest.archives[self.name] = (new_id, metadata.time) @@ -1813,6 +1816,19 @@ class ArchiveChecker: except msgpack.UnpackException: continue if valid_archive(archive): + # **after** doing the low-level checks and having a strong indication that we + # are likely looking at an archive item here, also check the TAM authentication: + try: + archive, verified, _ = self.key.unpack_and_verify_archive(data, force_tam_not_required=False) + except IntegrityError: + # TAM issues - do not accept this archive! + # either somebody is trying to attack us with a fake archive data or + # we have an ancient archive made before TAM was a thing (borg < 1.0.9) **and** this repo + # was not correctly upgraded to borg 1.2.5 (see advisory at top of the changelog). + # borg can't tell the difference, so it has to assume this archive might be an attack + # and drops this archive. + continue + # note: if we get here and verified is False, a TAM is not required. archive = ArchiveItem(internal_dict=archive) name = archive.name logger.info('Found archive %s', name) @@ -2048,7 +2064,17 @@ class ArchiveChecker: self.error_found = True del self.manifest.archives[info.name] continue - archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) + try: + archive, verified, salt = self.key.unpack_and_verify_archive(data, force_tam_not_required=False) + except IntegrityError as integrity_error: + # looks like there is a TAM issue with this archive, this might be an attack! + # when upgrading to borg 1.2.5, users are expected to TAM-authenticate all archives they + # trust, so there shouldn't be any without TAM. + logger.error('Archive TAM authentication issue for archive %s: %s', info.name, integrity_error) + self.error_found = True + del self.manifest.archives[info.name] + continue + archive = ArchiveItem(internal_dict=archive) if archive.version != 1: raise Exception('Unknown archive metadata version') archive.cmdline = [safe_decode(arg) for arg in archive.cmdline] @@ -2062,7 +2088,7 @@ class ArchiveChecker: for previous_item_id in archive.items: mark_as_possibly_superseded(previous_item_id) archive.items = items_buffer.chunks - data = msgpack.packb(archive.as_dict()) + data = self.key.pack_and_authenticate_metadata(archive.as_dict(), context=b'archive', salt=salt) new_archive_id = self.key.id_hash(data) cdata = self.key.encrypt(data) add_reference(new_archive_id, len(data), len(cdata), cdata) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 922ed483a..abf4fbb30 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -75,11 +75,11 @@ try: from .helpers import sig_int, ignore_sigint from .helpers import iter_separated from .helpers import get_tar_filter - from .helpers.parseformat import BorgJsonEncoder + from .helpers.parseformat import BorgJsonEncoder, safe_decode from .nanorst import rst_to_terminal from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern from .patterns import PatternMatcher - from .item import Item + from .item import Item, ArchiveItem from .platform import get_flags, get_process_id, SyncFile from .platform import uid2user, gid2group from .remote import RepositoryServer, RemoteRepository, cache_if_remote @@ -1618,25 +1618,43 @@ class Archiver: DASHES, logger=logging.getLogger('borg.output.stats')) return self.exit_code - @with_repository(fake=('tam', 'disable_tam'), invert_fake=True, manifest=False, exclusive=True) + @with_repository(fake=('tam', 'disable_tam', 'archives_tam'), invert_fake=True, manifest=False, exclusive=True) def do_upgrade(self, args, repository, manifest=None, key=None): """upgrade a repository from a previous version""" - if args.tam: + if args.archives_tam: + manifest, key = Manifest.load(repository, (Manifest.Operation.CHECK,), force_tam_not_required=args.force) + with Cache(repository, key, manifest) as cache: + stats = Statistics() + for info in manifest.archives.list(sort_by=['ts']): + archive_id = info.id + archive_formatted = format_archive(info) + cdata = repository.get(archive_id) + data = key.decrypt(archive_id, cdata) + archive, verified, _ = key.unpack_and_verify_archive(data, force_tam_not_required=True) + if not verified: # we do not have an archive TAM yet -> add TAM now! + archive = ArchiveItem(internal_dict=archive) + archive.cmdline = [safe_decode(arg) for arg in archive.cmdline] + data = key.pack_and_authenticate_metadata(archive.as_dict(), context=b'archive') + new_archive_id = key.id_hash(data) + cache.add_chunk(new_archive_id, data, stats) + cache.chunk_decref(archive_id, stats) + manifest.archives[info.name] = (new_archive_id, info.ts) + print(f"Added archive TAM: {archive_formatted} -> [{bin_to_hex(new_archive_id)}]") + else: + print(f"Archive TAM present: {archive_formatted}") + manifest.write() + repository.commit(compact=False) + cache.commit() + elif args.tam: manifest, key = Manifest.load(repository, (Manifest.Operation.CHECK,), force_tam_not_required=args.force) - - if not hasattr(key, 'change_passphrase'): - print('This repository is not encrypted, cannot enable TAM.') - return EXIT_ERROR - if not manifest.tam_verified or not manifest.config.get(b'tam_required', False): - # The standard archive listing doesn't include the archive ID like in borg 1.1.x print('Manifest contents:') for archive_info in manifest.archives.list(sort_by=['ts']): - print(format_archive(archive_info), '[%s]' % bin_to_hex(archive_info.id)) + print(format_archive(archive_info)) manifest.config[b'tam_required'] = True manifest.write() repository.commit(compact=False) - if not key.tam_required: + if not key.tam_required and hasattr(key, 'change_passphrase'): key.tam_required = True key.change_passphrase(key._passphrase) print('Key updated') @@ -1650,7 +1668,7 @@ class Archiver: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK, force_tam_not_required=True) if tam_required(repository): os.unlink(tam_required_file(repository)) - if key.tam_required: + if key.tam_required and hasattr(key, 'change_passphrase'): key.tam_required = False key.change_passphrase(key._passphrase) print('Key updated') @@ -4862,6 +4880,23 @@ class Archiver: Borg 1.x.y upgrades +++++++++++++++++++ + Archive TAM authentication: + + Use ``borg upgrade --archives-tam REPO`` to add archive TAMs to all + archives that are not TAM authenticated yet. + This is a convenient method to just trust all archives present - if + an archive does not have TAM authentication yet, a TAM will be added. + Archives created by old borg versions < 1.0.9 do not have TAMs. + Archives created by newer borg version should have TAMs already. + If you have a high risk environment, you should not just run this, + but first verify that the archives are authentic and not malicious + (== have good content, have a good timestamp). + Borg 1.2.5+ needs all archives to be TAM authenticated for safety reasons. + + This upgrade needs to be done once per repository. + + Manifest TAM authentication: + Use ``borg upgrade --tam REPO`` to require manifest authentication introduced with Borg 1.0.9 to address security issues. This means that modifying the repository after doing this with a version prior @@ -4942,6 +4977,8 @@ class Archiver: help='Enable manifest authentication (in key and cache) (Borg 1.0.9 and later).') subparser.add_argument('--disable-tam', dest='disable_tam', action='store_true', help='Disable manifest authentication (in key and cache).') + subparser.add_argument('--archives-tam', dest='archives_tam', action='store_true', + help='add TAM authentication for all archives.') subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='path to the repository to be upgraded') diff --git a/src/borg/cache.py b/src/borg/cache.py index b0ff5e541..523aedf93 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -755,7 +755,8 @@ class LocalCache(CacheStatsMixin): nonlocal processed_item_metadata_chunks csize, data = decrypted_repository.get(archive_id) chunk_idx.add(archive_id, 1, len(data), csize) - archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) + archive, verified, _ = self.key.unpack_and_verify_archive(data, force_tam_not_required=True) + archive = ArchiveItem(internal_dict=archive) if archive.version != 1: raise Exception('Unknown archive metadata version') sync = CacheSynchronizer(chunk_idx) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index ac998854c..c1ff76b1d 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -89,6 +89,13 @@ class TAMRequiredError(IntegrityError): traceback = False +class ArchiveTAMRequiredError(TAMRequiredError): + __doc__ = textwrap.dedent(""" + Archive '{}' is unauthenticated, but it is required for this repository. + """).strip() + traceback = False + + class TAMInvalid(IntegrityError): __doc__ = IntegrityError.__doc__ traceback = False @@ -98,6 +105,15 @@ class TAMInvalid(IntegrityError): super().__init__('Manifest authentication did not verify') +class ArchiveTAMInvalid(IntegrityError): + __doc__ = IntegrityError.__doc__ + traceback = False + + def __init__(self): + # Error message becomes: "Data integrity error: Archive authentication did not verify" + super().__init__('Archive authentication did not verify') + + class TAMUnsupportedSuiteError(IntegrityError): """Could not verify manifest: Unsupported suite {!r}; a newer version is needed.""" traceback = False @@ -210,15 +226,17 @@ class KeyBase: output_length=64 ) - def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'): + def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest', salt=None): + if salt is None: + salt = os.urandom(64) metadata_dict = StableDict(metadata_dict) tam = metadata_dict['tam'] = StableDict({ 'type': 'HKDF_HMAC_SHA512', 'hmac': bytes(64), - 'salt': os.urandom(64), + 'salt': salt, }) packed = msgpack.packb(metadata_dict) - tam_key = self._tam_key(tam['salt'], context) + tam_key = self._tam_key(salt, context) tam['hmac'] = hmac.digest(tam_key, packed, 'sha512') return msgpack.packb(metadata_dict) @@ -241,7 +259,7 @@ class KeyBase: if tam_required: raise TAMRequiredError(self.repository._location.canonical_path()) else: - logger.debug('TAM not found and not required') + logger.debug('Manifest TAM not found and not required') return unpacked, False tam = unpacked.pop(b'tam', None) if not isinstance(tam, dict): @@ -251,7 +269,7 @@ class KeyBase: if tam_required: raise TAMUnsupportedSuiteError(repr(tam_type)) else: - logger.debug('Ignoring TAM made with unsupported suite, since TAM is not required: %r', tam_type) + logger.debug('Ignoring manifest TAM made with unsupported suite, since TAM is not required: %r', tam_type) return unpacked, False tam_hmac = tam.get(b'hmac') tam_salt = tam.get(b'salt') @@ -266,6 +284,48 @@ class KeyBase: logger.debug('TAM-verified manifest') return unpacked, True + def unpack_and_verify_archive(self, data, force_tam_not_required=False): + """Unpack msgpacked *data* and return (object, did_verify).""" + tam_required = self.tam_required + if force_tam_not_required and tam_required: + # for a long time, borg only checked manifest for "tam_required" and + # people might have archives without TAM, so don't be too annoyingly loud here: + logger.debug('Archive authentication DISABLED.') + tam_required = False + data = bytearray(data) + unpacker = get_limited_unpacker('archive') + unpacker.feed(data) + unpacked = unpacker.unpack() + if b'tam' not in unpacked: + if tam_required: + archive_name = unpacked.get(b'name', b'').decode('ascii', 'replace') + raise ArchiveTAMRequiredError(archive_name) + else: + logger.debug('Archive TAM not found and not required') + return unpacked, False, None + tam = unpacked.pop(b'tam', None) + if not isinstance(tam, dict): + raise ArchiveTAMInvalid() + tam_type = tam.get(b'type', b'').decode('ascii', 'replace') + if tam_type != 'HKDF_HMAC_SHA512': + if tam_required: + raise TAMUnsupportedSuiteError(repr(tam_type)) + else: + logger.debug('Ignoring archive TAM made with unsupported suite, since TAM is not required: %r', tam_type) + return unpacked, False, None + tam_hmac = tam.get(b'hmac') + tam_salt = tam.get(b'salt') + if not isinstance(tam_salt, bytes) or not isinstance(tam_hmac, bytes): + raise ArchiveTAMInvalid() + offset = data.index(tam_hmac) + data[offset:offset + 64] = bytes(64) + tam_key = self._tam_key(tam_salt, context=b'archive') + calculated_hmac = hmac.digest(tam_key, data, 'sha512') + if not hmac.compare_digest(calculated_hmac, tam_hmac): + raise ArchiveTAMInvalid() + logger.debug('TAM-verified archive') + return unpacked, True, tam_salt + class PlaintextKey(KeyBase): TYPE = 0x02 diff --git a/src/borg/helpers/msgpack.py b/src/borg/helpers/msgpack.py index 3c6565639..309c98834 100644 --- a/src/borg/helpers/msgpack.py +++ b/src/borg/helpers/msgpack.py @@ -209,6 +209,12 @@ def get_limited_unpacker(kind): max_str_len=255, # archive name object_hook=StableDict, )) + elif kind == 'archive': + args.update(dict(use_list=True, # default value + max_map_len=100, # ARCHIVE_KEYS ~= 20 + max_str_len=10000, # comment + object_hook=StableDict, + )) elif kind == 'key': args.update(dict(use_list=True, # default value max_array_len=0, # not used diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index d96637b4d..1e11ea61d 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -592,9 +592,10 @@ class ArchiveFormatter(BaseFormatter): 'id': 'internal ID of the archive', 'hostname': 'hostname of host on which this archive was created', 'username': 'username of user who created this archive', + 'tam': 'TAM authentication state of this archive', } KEY_GROUPS = ( - ('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'), + ('archive', 'name', 'barchive', 'comment', 'bcomment', 'id', 'tam'), ('start', 'time', 'end', 'command_line'), ('hostname', 'username'), ) @@ -647,6 +648,7 @@ class ArchiveFormatter(BaseFormatter): 'bcomment': partial(self.get_meta, 'comment', rs=False), 'end': self.get_ts_end, 'command_line': self.get_cmdline, + 'tam': self.get_tam, } self.used_call_keys = set(self.call_keys) & self.format_keys if self.json: @@ -697,6 +699,9 @@ class ArchiveFormatter(BaseFormatter): def get_ts_end(self): return self.format_time(self.archive.ts_end) + def get_tam(self): + return 'verified' if self.archive.tam_verified else 'none' + def format_time(self, ts): return OutputTimestamp(ts) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index c586d2e25..8cc930b6f 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -36,11 +36,11 @@ from ..cache import Cache, LocalCache from ..chunker import has_seek_hole from ..constants import * # NOQA from ..crypto.low_level import bytes_to_long, num_cipher_blocks -from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError +from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError, ArchiveTAMRequiredError from ..crypto.keymanager import RepoIdMismatch, NotABorgKeyFile from ..crypto.file_integrity import FileIntegrityError from ..helpers import Location, get_security_dir -from ..helpers import Manifest, MandatoryFeatureUnsupported +from ..helpers import Manifest, MandatoryFeatureUnsupported, ArchiveInfo from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from ..helpers import bin_to_hex from ..helpers import MAX_S @@ -3957,7 +3957,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): corrupted_manifest = manifest + b'corrupted!' repository.put(Manifest.MANIFEST_ID, corrupted_manifest) - archive = msgpack.packb({ + archive_dict = { 'cmdline': [], 'items': [], 'hostname': 'foo', @@ -3965,7 +3965,8 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): 'name': 'archive1', 'time': '2016-12-15T18:49:51.849711', 'version': 1, - }) + } + archive = key.pack_and_authenticate_metadata(archive_dict, context=b'archive') archive_id = key.id_hash(archive) repository.put(archive_id, key.encrypt(archive)) repository.commit(compact=False) @@ -4094,7 +4095,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase): repository.commit(compact=False) output = self.cmd('list', '--debug', self.repository_location) assert 'archive1234' in output - assert 'TAM not found and not required' in output + assert 'Manifest TAM not found and not required' in output # Run upgrade self.cmd('upgrade', '--tam', self.repository_location) # Manifest must be authenticated now @@ -4127,6 +4128,70 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase): assert not self.cmd('list', self.repository_location) +class ArchiveAuthenticationTest(ArchiverTestCaseBase): + + def write_archive_without_tam(self, repository, archive_name): + manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + archive_data = msgpack.packb({ + 'version': 1, + 'name': archive_name, + 'items': [], + 'cmdline': '', + 'hostname': '', + 'username': '', + 'time': utcnow().strftime(ISO_FORMAT), + }) + archive_id = key.id_hash(archive_data) + repository.put(archive_id, key.encrypt(archive_data)) + manifest.archives[archive_name] = (archive_id, datetime.now()) + manifest.write() + repository.commit(compact=False) + + def test_upgrade_archives_tam(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + self.create_src_archive('archive_tam') + repository = Repository(self.repository_path, exclusive=True) + with repository: + self.write_archive_without_tam(repository, "archive_no_tam") + output = self.cmd('list', '--format="{name} tam:{tam}{NL}"', self.repository_location) + assert 'archive_tam tam:verified' in output # good + assert 'archive_no_tam tam:none' in output # could be borg < 1.0.9 archive or fake + self.cmd('upgrade', '--archives-tam', self.repository_location) + output = self.cmd('list', '--format="{name} tam:{tam}{NL}"', self.repository_location) + assert 'archive_tam tam:verified' in output # still good + assert 'archive_no_tam tam:verified' in output # previously TAM-less archives got a TAM now + + def test_check_rebuild_manifest(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + self.create_src_archive('archive_tam') + repository = Repository(self.repository_path, exclusive=True) + with repository: + self.write_archive_without_tam(repository, "archive_no_tam") + repository.delete(Manifest.MANIFEST_ID) # kill manifest, so check has to rebuild it + repository.commit(compact=False) + self.cmd('check', '--repair', self.repository_location) + output = self.cmd('list', '--format="{name} tam:{tam}{NL}"', self.repository_location) + assert 'archive_tam tam:verified' in output # TAM-verified archive is in rebuilt manifest + assert 'archive_no_tam' not in output # check got rid of untrusted not TAM-verified archive + + def test_check_rebuild_refcounts(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + self.create_src_archive('archive_tam') + archive_id_pre_check = self.cmd('list', '--format="{name} {id}{NL}"', self.repository_location) + repository = Repository(self.repository_path, exclusive=True) + with repository: + self.write_archive_without_tam(repository, "archive_no_tam") + output = self.cmd('list', '--format="{name} tam:{tam}{NL}"', self.repository_location) + assert 'archive_tam tam:verified' in output # good + assert 'archive_no_tam tam:none' in output # could be borg < 1.0.9 archive or fake + self.cmd('check', '--repair', self.repository_location) + output = self.cmd('list', '--format="{name} tam:{tam}{NL}"', self.repository_location) + assert 'archive_tam tam:verified' in output # TAM-verified archive still there + assert 'archive_no_tam' not in output # check got rid of untrusted not TAM-verified archive + archive_id_post_check = self.cmd('list', '--format="{name} {id}{NL}"', self.repository_location) + assert archive_id_post_check == archive_id_pre_check # rebuild_refcounts didn't change archive_tam archive id + + class RemoteArchiverTestCase(ArchiverTestCase): prefix = '__testsuite__:' diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 8107c2f09..5bc28958a 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -11,6 +11,7 @@ from ..crypto.key import PlaintextKey, PassphraseKey, AuthenticatedKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, Blake2AuthenticatedKey from ..crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256 from ..crypto.key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError +from ..crypto.key import ArchiveTAMInvalid from ..crypto.key import identify_key from ..crypto.low_level import bytes_to_long from ..crypto.low_level import IntegrityError as IntegrityErrorBase @@ -338,6 +339,8 @@ class TestTAM: blob = msgpack.packb({}) with pytest.raises(TAMRequiredError): key.unpack_and_verify_manifest(blob) + with pytest.raises(TAMRequiredError): + key.unpack_and_verify_archive(blob) def test_missing(self, key): blob = msgpack.packb({}) @@ -345,6 +348,9 @@ class TestTAM: unpacked, verified = key.unpack_and_verify_manifest(blob) assert unpacked == {} assert not verified + unpacked, verified, _ = key.unpack_and_verify_archive(blob) + assert unpacked == {} + assert not verified def test_unknown_type_when_required(self, key): blob = msgpack.packb({ @@ -354,6 +360,8 @@ class TestTAM: }) with pytest.raises(TAMUnsupportedSuiteError): key.unpack_and_verify_manifest(blob) + with pytest.raises(TAMUnsupportedSuiteError): + key.unpack_and_verify_archive(blob) def test_unknown_type(self, key): blob = msgpack.packb({ @@ -365,6 +373,9 @@ class TestTAM: unpacked, verified = key.unpack_and_verify_manifest(blob) assert unpacked == {} assert not verified + unpacked, verified, _ = key.unpack_and_verify_archive(blob) + assert unpacked == {} + assert not verified @pytest.mark.parametrize('tam, exc', ( ({}, TAMUnsupportedSuiteError), @@ -372,13 +383,26 @@ class TestTAM: (None, TAMInvalid), (1234, TAMInvalid), )) - def test_invalid(self, key, tam, exc): + def test_invalid_manifest(self, key, tam, exc): blob = msgpack.packb({ 'tam': tam, }) with pytest.raises(exc): key.unpack_and_verify_manifest(blob) + @pytest.mark.parametrize('tam, exc', ( + ({}, TAMUnsupportedSuiteError), + ({'type': b'\xff'}, TAMUnsupportedSuiteError), + (None, ArchiveTAMInvalid), + (1234, ArchiveTAMInvalid), + )) + def test_invalid_archive(self, key, tam, exc): + blob = msgpack.packb({ + 'tam': tam, + }) + with pytest.raises(exc): + key.unpack_and_verify_archive(blob) + @pytest.mark.parametrize('hmac, salt', ( ({}, bytes(64)), (bytes(64), {}), @@ -401,10 +425,12 @@ class TestTAM: blob = msgpack.packb(data) with pytest.raises(TAMInvalid): key.unpack_and_verify_manifest(blob) + with pytest.raises(ArchiveTAMInvalid): + key.unpack_and_verify_archive(blob) - def test_round_trip(self, key): + def test_round_trip_manifest(self, key): data = {'foo': 'bar'} - blob = key.pack_and_authenticate_metadata(data) + blob = key.pack_and_authenticate_metadata(data, context=b"manifest") assert blob.startswith(b'\x82') unpacked = msgpack.unpackb(blob) @@ -415,10 +441,23 @@ class TestTAM: assert unpacked[b'foo'] == b'bar' assert b'tam' not in unpacked - @pytest.mark.parametrize('which', (b'hmac', b'salt')) - def test_tampered(self, key, which): + def test_round_trip_archive(self, key): data = {'foo': 'bar'} - blob = key.pack_and_authenticate_metadata(data) + blob = key.pack_and_authenticate_metadata(data, context=b"archive") + assert blob.startswith(b'\x82') + + unpacked = msgpack.unpackb(blob) + assert unpacked[b'tam'][b'type'] == b'HKDF_HMAC_SHA512' + + unpacked, verified, _ = key.unpack_and_verify_archive(blob) + assert verified + assert unpacked[b'foo'] == b'bar' + assert b'tam' not in unpacked + + @pytest.mark.parametrize('which', (b'hmac', b'salt')) + def test_tampered_manifest(self, key, which): + data = {'foo': 'bar'} + blob = key.pack_and_authenticate_metadata(data, context=b"manifest") assert blob.startswith(b'\x82') unpacked = msgpack.unpackb(blob, object_hook=StableDict) @@ -429,3 +468,18 @@ class TestTAM: with pytest.raises(TAMInvalid): key.unpack_and_verify_manifest(blob) + + @pytest.mark.parametrize('which', (b'hmac', b'salt')) + def test_tampered_archive(self, key, which): + data = {'foo': 'bar'} + blob = key.pack_and_authenticate_metadata(data, context=b"archive") + assert blob.startswith(b'\x82') + + unpacked = msgpack.unpackb(blob, object_hook=StableDict) + assert len(unpacked[b'tam'][which]) == 64 + unpacked[b'tam'][which] = unpacked[b'tam'][which][0:32] + bytes(32) + assert len(unpacked[b'tam'][which]) == 64 + blob = msgpack.packb(unpacked) + + with pytest.raises(ArchiveTAMInvalid): + key.unpack_and_verify_archive(blob)