mirror of
https://github.com/borgbackup/borg.git
synced 2026-05-23 10:36:32 -04:00
Merge 854b6b2773 into e016a51cf2
This commit is contained in:
TW
GitHub
commit
a6c28fe84f
12 changed files with 261 additions and 166 deletions
31
.github/workflows/ci.yml
vendored
31
.github/workflows/ci.yml
vendored
|
|
@ -413,6 +413,8 @@ jobs:
|
|||
version: 'r1beta5'
|
||||
display_name: Haiku
|
||||
do_binaries: false
|
||||
# custom image with all packages updated, python, rust
|
||||
image_url: https://github.com/ThomasWaldmann/haiku-builder/releases/download/v0.1.4/haiku-r1beta5-x86-64.qcow2
|
||||
|
||||
steps:
|
||||
- name: Check out repository
|
||||
|
|
@ -430,6 +432,7 @@ jobs:
|
|||
operating_system: ${{ matrix.os }}
|
||||
version: ${{ matrix.version }}
|
||||
shell: bash
|
||||
image_url: ${{ matrix.image_url }}
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
|
||||
|
|
@ -537,7 +540,7 @@ jobs:
|
|||
;;
|
||||
|
||||
omnios)
|
||||
sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils
|
||||
sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils rust
|
||||
sudo ln -sf /usr/bin/python3.13 /usr/bin/python3
|
||||
sudo ln -sf /usr/bin/python3.13-config /usr/bin/python3-config
|
||||
sudo python3 -m ensurepip
|
||||
|
|
@ -557,16 +560,20 @@ jobs:
|
|||
|
||||
haiku)
|
||||
pkgman refresh
|
||||
pkgman install -y git pkgconfig lz4
|
||||
pkgman install -y openssl3
|
||||
pkgman install -y rust_bin
|
||||
pkgman install -y python3.10
|
||||
pkgman install -y cffi
|
||||
pkgman install -y lz4_devel openssl3_devel libffi_devel
|
||||
# pkgman install -y git
|
||||
pkgman install -y pkgconfig
|
||||
pkgman install -y lz4 lz4_devel
|
||||
# pkgman update -y openssl3 openssl3_devel
|
||||
# pkgman install -y python3.10
|
||||
# pkgman install -y rust_bin
|
||||
# pkgman install -y setuptools_rust setuptools_rust_python310 # not sure whether this is needed
|
||||
pkgman install -y cffi libffi_devel
|
||||
|
||||
cargo --version || true
|
||||
|
||||
# there is no pkgman package for tox, so we install it into a venv
|
||||
python3 -m ensurepip --upgrade
|
||||
python3 -m pip install --upgrade pip wheel
|
||||
python3 -m pip install --upgrade pip setuptools wheel
|
||||
python3 -m venv .venv
|
||||
. .venv/bin/activate
|
||||
|
||||
|
|
@ -574,6 +581,14 @@ jobs:
|
|||
export BORG_LIBLZ4_PREFIX=/system/develop
|
||||
export BORG_OPENSSL_PREFIX=/system/develop
|
||||
pip install -r requirements.d/development.lock.txt
|
||||
mkdir -p .cargo
|
||||
echo "[build]" > .cargo/config.toml
|
||||
echo "jobs = 1" >> .cargo/config.toml
|
||||
echo "" >> .cargo/config.toml
|
||||
echo "[profile.release]" >> .cargo/config.toml
|
||||
echo "codegen-units = 1" >> .cargo/config.toml
|
||||
pip install -v maturin
|
||||
pip install -v blake3
|
||||
pip install -e .
|
||||
|
||||
# troubles with either tox or pytest xdist, so we run pytest manually:
|
||||
|
|
|
|||
|
|
@ -2,52 +2,88 @@
|
|||
|
||||
Examples
|
||||
~~~~~~~~
|
||||
|
||||
To keep the following examples short and readable, we export the repository
|
||||
locations and passphrases first:
|
||||
|
||||
::
|
||||
|
||||
# 0. Have Borg 2.0 installed on the client AND server; have a b12 repository copy for testing.
|
||||
export BORG_REPO=ssh://borg2@borgbackup/./tests/b20
|
||||
export BORG_PASSPHRASE='your-borg2-repo-passphrase'
|
||||
export BORG_OTHER_REPO=ssh://borg2@borgbackup/./tests/b1x
|
||||
export BORG_OTHER_PASSPHRASE='your-borg1-repo-passphrase'
|
||||
|
||||
::
|
||||
|
||||
# Borg 1.x repository -> Borg 2.0 repository (hmac-sha256 -> hmac-sha256, keeping the same chunk ID algorithm)
|
||||
|
||||
# 0. Have Borg 2.0 installed on the client AND server; have a Borg 1.x repository copy for testing.
|
||||
|
||||
# 1. Create a new "related" repository:
|
||||
# Here, the existing Borg 1.2 repository used repokey-blake2 (and AES-CTR mode),
|
||||
# thus we use repokey-blake2-aes-ocb for the new Borg 2.0 repository.
|
||||
# Staying with the same chunk ID algorithm (BLAKE2) and with the same
|
||||
# key material (via --other-repo <oldrepo>) will make deduplication work
|
||||
# Here, the existing Borg 1.x repository used repokey (and AES-CTR mode),
|
||||
# thus we use repokey-aes-ocb for the new Borg 2.0 repository.
|
||||
# Staying with the same chunk ID algorithm (hmac-sha256) and with the same
|
||||
# key material (via BORG_OTHER_REPO) will make deduplication work
|
||||
# between old archives (copied with borg transfer) and future ones.
|
||||
# The AEAD cipher does not matter (everything must be re-encrypted and
|
||||
# re-authenticated anyway); you could also choose repokey-blake2-chacha20-poly1305.
|
||||
# In case your old Borg repository did not use BLAKE2, just remove the "-blake2".
|
||||
$ borg --repo ssh://borg2@borgbackup/./tests/b20 repo-create \
|
||||
--other-repo ssh://borg2@borgbackup/./tests/b12 -e repokey-blake2-aes-ocb
|
||||
# re-authenticated anyway); you could also choose repokey-chacha20-poly1305.
|
||||
$ borg repo-create -e repokey-aes-ocb
|
||||
|
||||
# 2. Check what and how much it would transfer:
|
||||
$ borg --repo ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
|
||||
--other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
|
||||
$ borg transfer --from-borg1 --dry-run
|
||||
|
||||
# 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!):
|
||||
$ borg --repo ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
|
||||
--other-repo ssh://borg2@borgbackup/./tests/b12
|
||||
$ borg transfer --from-borg1
|
||||
|
||||
# 4. Check whether we have everything (same as step 2):
|
||||
$ borg --repo ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
|
||||
--other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
|
||||
$ borg transfer --from-borg1 --dry-run
|
||||
|
||||
Keyfile considerations when upgrading from borg 1.x
|
||||
::
|
||||
|
||||
# Borg 1.x repository -> Borg 2.0 repository (blake2 -> blake3, changing the chunk ID algorithm)
|
||||
|
||||
# 0. Have Borg 2.0 installed on the client AND server; have a Borg 1.x repository copy for testing.
|
||||
|
||||
# 1. Create a new "related" repository:
|
||||
# Here, the existing Borg 1.x repository used repokey-blake2 (and AES-CTR mode),
|
||||
# thus we use repokey-blake3-aes-ocb for the new Borg 2.0 repository.
|
||||
# We need to change from blake2 to blake3, because blake2 is not supported
|
||||
# for borg2 repos (blake3 is much faster). Because we change how chunk IDs are
|
||||
# computed, we need to re-chunk everything while doing the transfer.
|
||||
# The chunker parameters you provide here should be the same as you will
|
||||
# use for all future Borg 2.0 archives.
|
||||
# The AEAD cipher does not matter (everything must be re-encrypted and
|
||||
# re-authenticated anyway); you could also choose repokey-blake3-chacha20-poly1305.
|
||||
$ borg repo-create -e repokey-blake3-aes-ocb
|
||||
$ export CHUNKER_PARAMS="buzhash64,19,23,21,4095"
|
||||
|
||||
# 2. Check what and how much it would transfer:
|
||||
$ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run
|
||||
|
||||
# 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!):
|
||||
$ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS
|
||||
|
||||
# 4. Check whether we have everything (same as step 2):
|
||||
$ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run
|
||||
|
||||
Keyfile considerations when upgrading from Borg 1.x
|
||||
++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
If you are using a ``keyfile`` encryption mode (not ``repokey``), borg 2
|
||||
may not automatically find your borg 1.x key file, because the default
|
||||
If you are using a ``keyfile`` encryption mode (not ``repokey``), Borg 2
|
||||
may not automatically find your Borg 1.x key file, because the default
|
||||
key file directory has changed on some platforms due to the switch to
|
||||
the `platformdirs <https://pypi.org/project/platformdirs/>`_ library.
|
||||
|
||||
On **Linux**, there is typically no change -- both borg 1.x and borg 2
|
||||
On **Linux**, there is typically no change -- both Borg 1.x and Borg 2
|
||||
use ``~/.config/borg/keys/``.
|
||||
|
||||
On **macOS**, borg 1.x stored key files in ``~/.config/borg/keys/``,
|
||||
but borg 2 defaults to ``~/Library/Application Support/borg/keys/``.
|
||||
On **macOS**, Borg 1.x stored key files in ``~/.config/borg/keys/``,
|
||||
but Borg 2 defaults to ``~/Library/Application Support/borg/keys/``.
|
||||
|
||||
On **Windows**, borg 1.x used XDG-style paths (e.g. ``~/.config/borg/keys/``),
|
||||
while borg 2 defaults to ``C:\Users\<user>\AppData\Roaming\borg\keys\``.
|
||||
On **Windows**, Borg 1.x used XDG-style paths (e.g. ``~/.config/borg/keys/``),
|
||||
while Borg 2 defaults to ``C:\Users\<user>\AppData\Roaming\borg\keys\``.
|
||||
|
||||
If borg 2 cannot find your key file, you have several options:
|
||||
If Borg 2 cannot find your key file, you have several options:
|
||||
|
||||
1. **Copy the key file** from the old location to the new one.
|
||||
2. **Set BORG_KEYS_DIR** to point to the old key file directory::
|
||||
|
|
@ -58,13 +94,13 @@ If borg 2 cannot find your key file, you have several options:
|
|||
|
||||
export BORG_KEY_FILE=~/.config/borg/keys/your_key_file
|
||||
|
||||
4. **Set BORG_BASE_DIR** to force borg 2 to use the same base directory
|
||||
as borg 1.x::
|
||||
4. **Set BORG_BASE_DIR** to force Borg 2 to use the same base directory
|
||||
as Borg 1.x::
|
||||
|
||||
export BORG_BASE_DIR=$HOME
|
||||
|
||||
This makes borg 2 use ``$HOME/.config/borg``, ``$HOME/.cache/borg``,
|
||||
etc., matching borg 1.x behaviour on all platforms.
|
||||
This makes Borg 2 use ``$HOME/.config/borg``, ``$HOME/.cache/borg``,
|
||||
etc., matching Borg 1.x behavior on all platforms.
|
||||
|
||||
See :ref:`env_vars` for more details on directory environment variables.
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ dependencies = [
|
|||
"xxhash>=2.0.0",
|
||||
"jsonargparse>=4.47.0",
|
||||
"PyYAML>=6.0.2", # we need to register our types with yaml, jsonargparse uses yaml for config files
|
||||
"blake3>=1.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
|
||||
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin}
|
||||
|
||||
if [ "$1" = "development" ]; then
|
||||
pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}
|
||||
|
|
|
|||
|
|
@ -232,16 +232,20 @@ class BenchmarkMixIn:
|
|||
print(f"{spec:<24} {format_file_size(size):<10} {dt:.3f}s")
|
||||
|
||||
from ..crypto.low_level import hmac_sha256, blake2b_256
|
||||
import blake3
|
||||
|
||||
if not args.json:
|
||||
print("Cryptographic hashes / MACs ====================================")
|
||||
else:
|
||||
result["hashes"] = []
|
||||
size = 1000000000
|
||||
for spec, func in [
|
||||
hashes_tests = [
|
||||
("hmac-sha256", lambda: hmac_sha256(key_256, random_10M)),
|
||||
("blake2b-256", lambda: blake2b_256(key_256, random_10M)),
|
||||
]:
|
||||
("blake3", lambda: blake3.blake3(random_10M, key=key_256).digest()),
|
||||
("blake3-mt", lambda: blake3.blake3(random_10M, key=key_256, max_threads=blake3.blake3.AUTO).digest()),
|
||||
]
|
||||
for spec, func in hashes_tests:
|
||||
dt = timeit(func, number=number_default)
|
||||
if args.json:
|
||||
result["hashes"].append({"algo": spec, "size": size, "time": dt})
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import os
|
||||
|
||||
from ..constants import * # NOQA
|
||||
from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake2AESOCBRepoKey, Blake2CHPORepoKey
|
||||
from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake2AESOCBKeyfileKey, Blake2CHPOKeyfileKey
|
||||
from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake3AESOCBRepoKey, Blake3CHPORepoKey
|
||||
from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake3AESOCBKeyfileKey, Blake3CHPOKeyfileKey
|
||||
from ..crypto.keymanager import KeyManager
|
||||
from ..helpers import PathSpec, CommandError
|
||||
from ..helpers.argparsing import ArgumentParser
|
||||
|
|
@ -40,10 +40,10 @@ class KeysMixIn:
|
|||
key_new = AESOCBKeyfileKey(repository)
|
||||
elif isinstance(key, CHPORepoKey):
|
||||
key_new = CHPOKeyfileKey(repository)
|
||||
elif isinstance(key, Blake2AESOCBRepoKey):
|
||||
key_new = Blake2AESOCBKeyfileKey(repository)
|
||||
elif isinstance(key, Blake2CHPORepoKey):
|
||||
key_new = Blake2CHPOKeyfileKey(repository)
|
||||
elif isinstance(key, Blake3AESOCBRepoKey):
|
||||
key_new = Blake3AESOCBKeyfileKey(repository)
|
||||
elif isinstance(key, Blake3CHPORepoKey):
|
||||
key_new = Blake3CHPOKeyfileKey(repository)
|
||||
else:
|
||||
print("Change not needed or not supported.")
|
||||
return
|
||||
|
|
@ -52,10 +52,10 @@ class KeysMixIn:
|
|||
key_new = AESOCBRepoKey(repository)
|
||||
elif isinstance(key, CHPOKeyfileKey):
|
||||
key_new = CHPORepoKey(repository)
|
||||
elif isinstance(key, Blake2AESOCBKeyfileKey):
|
||||
key_new = Blake2AESOCBRepoKey(repository)
|
||||
elif isinstance(key, Blake2CHPOKeyfileKey):
|
||||
key_new = Blake2CHPORepoKey(repository)
|
||||
elif isinstance(key, Blake3AESOCBKeyfileKey):
|
||||
key_new = Blake3AESOCBRepoKey(repository)
|
||||
elif isinstance(key, Blake3CHPOKeyfileKey):
|
||||
key_new = Blake3CHPORepoKey(repository)
|
||||
else:
|
||||
print("Change not needed or not supported.")
|
||||
return
|
||||
|
|
|
|||
|
|
@ -135,12 +135,11 @@ class TransferMixIn:
|
|||
"""archives transfer from other repository, optionally upgrade data format"""
|
||||
key = manifest.key
|
||||
other_key = other_manifest.key
|
||||
if not uses_same_id_hash(other_key, key):
|
||||
raise Error(
|
||||
"You must keep the same ID hash ([HMAC-]SHA256 or BLAKE2b) or deduplication will break. "
|
||||
"Use a related repository!"
|
||||
)
|
||||
if not uses_same_chunker_secret(other_key, key):
|
||||
using_same_id_hash = uses_same_id_hash(other_key, key)
|
||||
rechunking = args.chunker_params is not None
|
||||
if not using_same_id_hash and not rechunking:
|
||||
raise Error("You must either keep the same ID hash or use --chunker-params.")
|
||||
if not rechunking and not uses_same_chunker_secret(other_key, key):
|
||||
raise Error(
|
||||
"You must use the same chunker secret or deduplication will break. " "Use a related repository!"
|
||||
)
|
||||
|
|
@ -185,7 +184,7 @@ class TransferMixIn:
|
|||
raise Error(f"No such upgrader: {upgrader}")
|
||||
|
||||
if UpgraderCls is not upgrade_mod.UpgraderFrom12To20 and other_manifest.repository.version == 1:
|
||||
raise Error("To transfer from a borg 1.x repo, you need to use: --upgrader=From12To20")
|
||||
raise Error("To transfer from a Borg 1.x repo, you need to use: --upgrader=From12To20")
|
||||
|
||||
upgrader = UpgraderCls(cache=cache, args=args)
|
||||
|
||||
|
|
@ -308,13 +307,13 @@ class TransferMixIn:
|
|||
borg --repo=DST_REPO transfer --other-repo=SRC_REPO # do it!
|
||||
borg --repo=DST_REPO transfer --other-repo=SRC_REPO --dry-run # check! anything left?
|
||||
|
||||
Data migration / upgrade from borg 1.x
|
||||
Data migration / upgrade from Borg 1.x
|
||||
++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
To migrate your borg 1.x archives into a related, new borg2 repository, usage is quite similar
|
||||
To migrate your Borg 1.x archives into a related, new Borg 2 repository, usage is quite similar
|
||||
to the above, but you need the ``--from-borg1`` option::
|
||||
|
||||
borg --repo=DST_REPO repocreate --encryption=DST_ENC --other-repo=SRC_REPO --from-borg1
|
||||
borg --repo=DST_REPO repo-create --encryption=DST_ENC --other-repo=SRC_REPO --from-borg1
|
||||
|
||||
# to continue using lz4 compression as you did in SRC_REPO:
|
||||
borg --repo=DST_REPO transfer --other-repo=SRC_REPO --from-borg1 \\
|
||||
|
|
@ -333,7 +332,7 @@ class TransferMixIn:
|
|||
subparser = ArgumentParser(
|
||||
parents=[common_parser], description=self.do_transfer.__doc__, epilog=transfer_epilog
|
||||
)
|
||||
subparsers.add_subcommand("transfer", subparser, help="transfer of archives from another repository")
|
||||
subparsers.add_subcommand("transfer", subparser, help="Transfer of archives from another repository")
|
||||
subparser.add_argument(
|
||||
"-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository, just check"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -191,10 +191,11 @@ class KeyType:
|
|||
AESOCBREPO = 0x11
|
||||
CHPOKEYFILE = 0x20
|
||||
CHPOREPO = 0x21
|
||||
BLAKE2AESOCBKEYFILE = 0x30
|
||||
BLAKE2AESOCBREPO = 0x31
|
||||
BLAKE2CHPOKEYFILE = 0x40
|
||||
BLAKE2CHPOREPO = 0x41
|
||||
BLAKE3AESOCBKEYFILE = 0x30
|
||||
BLAKE3AESOCBREPO = 0x31
|
||||
BLAKE3CHPOKEYFILE = 0x40
|
||||
BLAKE3CHPOREPO = 0x41
|
||||
BLAKE3AUTHENTICATED = 0x50
|
||||
|
||||
|
||||
CACHE_TAG_NAME = "CACHEDIR.TAG"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from ..logger import create_logger
|
|||
|
||||
logger = create_logger()
|
||||
|
||||
from blake3 import blake3
|
||||
import argon2.low_level
|
||||
|
||||
from ..constants import * # NOQA
|
||||
|
|
@ -28,11 +29,11 @@ from ..platform import SaveFile
|
|||
from ..repoobj import RepoObj
|
||||
|
||||
|
||||
from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256
|
||||
from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256
|
||||
from .low_level import AES256_OCB, CHACHA20_POLY1305
|
||||
from . import low_level
|
||||
|
||||
# workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode
|
||||
# workaround for lost passphrase or key in "authenticated*" modes
|
||||
AUTHENTICATED_NO_KEY = "authenticated_no_key" in workarounds
|
||||
|
||||
|
||||
|
|
@ -123,19 +124,19 @@ def uses_same_id_hash(other_key, key):
|
|||
new_sha256_ids = (PlaintextKey,)
|
||||
old_hmac_sha256_ids = (RepoKey, KeyfileKey, AuthenticatedKey)
|
||||
new_hmac_sha256_ids = (AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey, AuthenticatedKey)
|
||||
old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey, Blake2AuthenticatedKey)
|
||||
new_blake2_ids = (
|
||||
Blake2AESOCBRepoKey,
|
||||
Blake2AESOCBKeyfileKey,
|
||||
Blake2CHPORepoKey,
|
||||
Blake2CHPOKeyfileKey,
|
||||
Blake2AuthenticatedKey,
|
||||
# note: we do not support blake2b for new repos, see #8867
|
||||
new_blake3_ids = (
|
||||
Blake3AESOCBRepoKey,
|
||||
Blake3AESOCBKeyfileKey,
|
||||
Blake3CHPORepoKey,
|
||||
Blake3CHPOKeyfileKey,
|
||||
Blake3AuthenticatedKey,
|
||||
)
|
||||
same_ids = (
|
||||
isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids)
|
||||
and isinstance(key, new_hmac_sha256_ids)
|
||||
or isinstance(other_key, old_blake2_ids + new_blake2_ids)
|
||||
and isinstance(key, new_blake2_ids)
|
||||
or isinstance(other_key, new_blake3_ids)
|
||||
and isinstance(key, new_blake3_ids)
|
||||
or isinstance(other_key, old_sha256_ids + new_sha256_ids)
|
||||
and isinstance(key, new_sha256_ids)
|
||||
)
|
||||
|
|
@ -276,38 +277,6 @@ class PlaintextKey(KeyBase):
|
|||
return memoryview(data)[1:]
|
||||
|
||||
|
||||
def random_blake2b_256_key():
|
||||
# This might look a bit curious, but is the same construction used in the keyed mode of BLAKE2b.
|
||||
# Why limit the key to 64 bytes and pad it with 64 nulls nonetheless? The answer is that BLAKE2b
|
||||
# has a 128 byte block size, but only 64 bytes of internal state (this is also referred to as a
|
||||
# "local wide pipe" design, because the compression function transforms (block, state) => state,
|
||||
# and len(block) >= len(state), hence wide.)
|
||||
# In other words, a key longer than 64 bytes would have simply no advantage, since the function
|
||||
# has no way of propagating more than 64 bytes of entropy internally.
|
||||
# It's padded to a full block so that the key is never buffered internally by blake2b_update, ie.
|
||||
# it remains in a single memory location that can be tracked and could be erased securely, if we
|
||||
# wanted to.
|
||||
return os.urandom(64) + bytes(64)
|
||||
|
||||
|
||||
class ID_BLAKE2b_256:
|
||||
"""
|
||||
Key mix-in class for using BLAKE2b-256 for the id key.
|
||||
|
||||
The id_key length must be 32 bytes.
|
||||
"""
|
||||
|
||||
def id_hash(self, data):
|
||||
return blake2b_256(self.id_key, data)
|
||||
|
||||
def init_from_random_data(self):
|
||||
super().init_from_random_data()
|
||||
enc_key = os.urandom(32)
|
||||
enc_hmac_key = random_blake2b_256_key()
|
||||
self.crypt_key = enc_key + enc_hmac_key
|
||||
self.id_key = random_blake2b_256_key()
|
||||
|
||||
|
||||
class ID_HMAC_SHA_256:
|
||||
"""
|
||||
Key mix-in class for using HMAC-SHA-256 for the id key.
|
||||
|
|
@ -558,15 +527,18 @@ class FlexiKey:
|
|||
if isinstance(key, AESKeyBase):
|
||||
# user must use an AEADKeyBase subclass (AEAD modes with session keys)
|
||||
raise Error("Copying key material to an AES-CTR based mode is insecure and unsupported.")
|
||||
if not uses_same_id_hash(other_key, key):
|
||||
raise Error("You must keep the same ID hash (HMAC-SHA256 or BLAKE2b) or deduplication will break.")
|
||||
if other_key.copy_crypt_key:
|
||||
# give the user the option to use the same authenticated encryption (AE) key
|
||||
crypt_key = other_key.crypt_key
|
||||
else:
|
||||
# borg transfer re-encrypts all data anyway, thus we can default to a new, random AE key
|
||||
crypt_key = os.urandom(64)
|
||||
key.init_from_given_data(crypt_key=crypt_key, id_key=other_key.id_key, chunk_seed=other_key.chunk_seed)
|
||||
if len(other_key.id_key) == 128: # blake2b id key from borg 1.x is not supported anymore
|
||||
id_key = os.urandom(32) # hmac-sha256 and blake3 use 32 bytes
|
||||
else:
|
||||
id_key = other_key.id_key
|
||||
chunk_seed = other_key.chunk_seed
|
||||
key.init_from_given_data(crypt_key=crypt_key, id_key=id_key, chunk_seed=chunk_seed)
|
||||
else:
|
||||
key.init_from_random_data()
|
||||
passphrase = Passphrase.new(allow_empty=True)
|
||||
|
|
@ -729,12 +701,6 @@ class FlexiKey:
|
|||
raise TypeError("Unsupported borg key storage type")
|
||||
|
||||
|
||||
# legacy imports placed after FlexiKey/AESKeyBase/KeyBase so those names are already
|
||||
# in the partial module when legacy/crypto/key.py imports them back during circular load
|
||||
from ..legacy.crypto.key import KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey # noqa: E402
|
||||
from ..legacy.crypto.key import LEGACY_KEY_TYPES # noqa: E402
|
||||
|
||||
|
||||
class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
|
||||
|
|
@ -774,6 +740,14 @@ class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
|
|||
return memoryview(data)[1:]
|
||||
|
||||
|
||||
# legacy imports placed after FlexiKey/AESKeyBase/KeyBase/AuthenticatedKeyBase so those names are already
|
||||
# in the partial module when legacy/crypto/key.py imports them back during circular load
|
||||
from ..legacy.crypto.key import KeyfileKey, RepoKey
|
||||
from ..legacy.crypto.key import Blake2KeyfileKey, Blake2RepoKey, Blake2AuthenticatedKey # noqa: F401
|
||||
from ..legacy.crypto.key import LEGACY_KEY_TYPES # noqa: E402
|
||||
from ..legacy.crypto.key import ID_BLAKE2b_256 # noqa: F401
|
||||
|
||||
|
||||
class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
|
||||
TYPE = KeyType.AUTHENTICATED
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
|
|
@ -781,16 +755,27 @@ class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
|
|||
ARG_NAME = "authenticated"
|
||||
|
||||
|
||||
class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase):
|
||||
TYPE = KeyType.BLAKE2AUTHENTICATED
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
NAME = "authenticated BLAKE2b"
|
||||
ARG_NAME = "authenticated-blake2"
|
||||
|
||||
|
||||
# ------------ new crypto ------------
|
||||
|
||||
|
||||
class ID_BLAKE3_256:
|
||||
"""
|
||||
Key mix-in class for using BLAKE3 for the id key.
|
||||
|
||||
The id_key length must be 32 bytes.
|
||||
"""
|
||||
|
||||
def id_hash(self, data):
|
||||
return blake3(data, key=self.id_key).digest(length=32)
|
||||
|
||||
|
||||
class Blake3AuthenticatedKey(ID_BLAKE3_256, AuthenticatedKeyBase):
|
||||
TYPE = KeyType.BLAKE3AUTHENTICATED
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
NAME = "authenticated BLAKE3"
|
||||
ARG_NAME = "authenticated-blake3"
|
||||
|
||||
|
||||
class AEADKeyBase(KeyBase):
|
||||
"""
|
||||
Chunks are encrypted and authenticated using some AEAD ciphersuite
|
||||
|
|
@ -936,38 +921,38 @@ class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
|||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
|
||||
class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO}
|
||||
TYPE = KeyType.BLAKE2AESOCBKEYFILE
|
||||
NAME = "key file BLAKE2b AES-OCB"
|
||||
ARG_NAME = "keyfile-blake2-aes-ocb"
|
||||
class Blake3AESOCBKeyfileKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE3AESOCBKEYFILE, KeyType.BLAKE3AESOCBREPO}
|
||||
TYPE = KeyType.BLAKE3AESOCBKEYFILE
|
||||
NAME = "key file BLAKE3 AES-OCB"
|
||||
ARG_NAME = "keyfile-blake3-aes-ocb"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = AES256_OCB
|
||||
|
||||
|
||||
class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO}
|
||||
TYPE = KeyType.BLAKE2AESOCBREPO
|
||||
NAME = "repokey BLAKE2b AES-OCB"
|
||||
ARG_NAME = "repokey-blake2-aes-ocb"
|
||||
class Blake3AESOCBRepoKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE3AESOCBKEYFILE, KeyType.BLAKE3AESOCBREPO}
|
||||
TYPE = KeyType.BLAKE3AESOCBREPO
|
||||
NAME = "repokey BLAKE3 AES-OCB"
|
||||
ARG_NAME = "repokey-blake3-aes-ocb"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = AES256_OCB
|
||||
|
||||
|
||||
class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO}
|
||||
TYPE = KeyType.BLAKE2CHPOKEYFILE
|
||||
NAME = "key file BLAKE2b ChaCha20-Poly1305"
|
||||
ARG_NAME = "keyfile-blake2-chacha20-poly1305"
|
||||
class Blake3CHPOKeyfileKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE3CHPOKEYFILE, KeyType.BLAKE3CHPOREPO}
|
||||
TYPE = KeyType.BLAKE3CHPOKEYFILE
|
||||
NAME = "key file BLAKE3 ChaCha20-Poly1305"
|
||||
ARG_NAME = "keyfile-blake3-chacha20-poly1305"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
|
||||
class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO}
|
||||
TYPE = KeyType.BLAKE2CHPOREPO
|
||||
NAME = "repokey BLAKE2b ChaCha20-Poly1305"
|
||||
ARG_NAME = "repokey-blake2-chacha20-poly1305"
|
||||
class Blake3CHPORepoKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE3CHPOKEYFILE, KeyType.BLAKE3CHPOREPO}
|
||||
TYPE = KeyType.BLAKE3CHPOREPO
|
||||
NAME = "repokey BLAKE3 ChaCha20-Poly1305"
|
||||
ARG_NAME = "repokey-blake3-chacha20-poly1305"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
|
|
@ -977,14 +962,14 @@ AVAILABLE_KEY_TYPES = (
|
|||
# not encrypted modes
|
||||
PlaintextKey,
|
||||
AuthenticatedKey,
|
||||
Blake2AuthenticatedKey,
|
||||
# new crypto
|
||||
Blake3AuthenticatedKey,
|
||||
AESOCBKeyfileKey,
|
||||
AESOCBRepoKey,
|
||||
CHPOKeyfileKey,
|
||||
CHPORepoKey,
|
||||
Blake2AESOCBKeyfileKey,
|
||||
Blake2AESOCBRepoKey,
|
||||
Blake2CHPOKeyfileKey,
|
||||
Blake2CHPORepoKey,
|
||||
Blake3AESOCBKeyfileKey,
|
||||
Blake3AESOCBRepoKey,
|
||||
Blake3CHPOKeyfileKey,
|
||||
Blake3CHPORepoKey,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,47 @@
|
|||
import os
|
||||
|
||||
from ...constants import * # NOQA
|
||||
from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b
|
||||
from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, AESKeyBase, FlexiKey
|
||||
from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, blake2b_256
|
||||
from ...crypto.key import ID_HMAC_SHA_256, AESKeyBase, FlexiKey, AuthenticatedKeyBase
|
||||
|
||||
|
||||
def random_blake2b_256_key():
|
||||
# This might look a bit curious, but is the same construction used in the keyed mode of BLAKE2b.
|
||||
# Why limit the key to 64 bytes and pad it with 64 nulls nonetheless? The answer is that BLAKE2b
|
||||
# has a 128 byte block size, but only 64 bytes of internal state (this is also referred to as a
|
||||
# "local wide pipe" design, because the compression function transforms (block, state) => state,
|
||||
# and len(block) >= len(state), hence wide.)
|
||||
# In other words, a key longer than 64 bytes would have simply no advantage, since the function
|
||||
# has no way of propagating more than 64 bytes of entropy internally.
|
||||
# It's padded to a full block so that the key is never buffered internally by blake2b_update, ie.
|
||||
# it remains in a single memory location that can be tracked and could be erased securely, if we
|
||||
# wanted to.
|
||||
return os.urandom(64) + bytes(64)
|
||||
|
||||
|
||||
class ID_BLAKE2b_256:
|
||||
"""
|
||||
Key mix-in class for using BLAKE2b-256 for the id key.
|
||||
|
||||
The id_key length must be 32 bytes.
|
||||
"""
|
||||
|
||||
def id_hash(self, data):
|
||||
return blake2b_256(self.id_key, data)
|
||||
|
||||
def init_from_random_data(self):
|
||||
super().init_from_random_data()
|
||||
enc_key = os.urandom(32)
|
||||
enc_hmac_key = random_blake2b_256_key()
|
||||
self.crypt_key = enc_key + enc_hmac_key
|
||||
self.id_key = random_blake2b_256_key()
|
||||
|
||||
|
||||
class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase): # type: ignore[misc]
|
||||
TYPE = KeyType.BLAKE2AUTHENTICATED
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
NAME = "authenticated BLAKE2b"
|
||||
ARG_NAME = "authenticated-blake2"
|
||||
|
||||
|
||||
class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
|
||||
|
|
@ -39,4 +80,4 @@ class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc]
|
|||
CIPHERSUITE = AES256_CTR_BLAKE2b
|
||||
|
||||
|
||||
LEGACY_KEY_TYPES = (KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey)
|
||||
LEGACY_KEY_TYPES = (KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, Blake2AuthenticatedKey)
|
||||
|
|
|
|||
|
|
@ -36,14 +36,14 @@ def test_change_location_to_keyfile(archivers, request):
|
|||
assert "(key file" in log
|
||||
|
||||
|
||||
def test_change_location_to_b2keyfile(archivers, request):
|
||||
def test_change_location_to_b3keyfile(archivers, request):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", "--encryption=repokey-blake2-aes-ocb")
|
||||
cmd(archiver, "repo-create", "--encryption=repokey-blake3-aes-ocb")
|
||||
log = cmd(archiver, "repo-info")
|
||||
assert "(repokey BLAKE2b" in log
|
||||
assert "(repokey BLAKE3" in log
|
||||
cmd(archiver, "key", "change-location", "keyfile")
|
||||
log = cmd(archiver, "repo-info")
|
||||
assert "(key file BLAKE2b" in log
|
||||
assert "(key file BLAKE3" in log
|
||||
|
||||
|
||||
def test_change_location_to_repokey(archivers, request):
|
||||
|
|
@ -56,14 +56,14 @@ def test_change_location_to_repokey(archivers, request):
|
|||
assert "(repokey" in log
|
||||
|
||||
|
||||
def test_change_location_to_b2repokey(archivers, request):
|
||||
def test_change_location_to_b3repokey(archivers, request):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", "--encryption=keyfile-blake2-aes-ocb")
|
||||
cmd(archiver, "repo-create", "--encryption=keyfile-blake3-aes-ocb")
|
||||
log = cmd(archiver, "repo-info")
|
||||
assert "(key file BLAKE2b" in log
|
||||
assert "(key file BLAKE3" in log
|
||||
cmd(archiver, "key", "change-location", "repokey")
|
||||
log = cmd(archiver, "repo-info")
|
||||
assert "(repokey BLAKE2b" in log
|
||||
assert "(repokey BLAKE3" in log
|
||||
|
||||
|
||||
def test_key_export_keyfile(archivers, request):
|
||||
|
|
|
|||
|
|
@ -8,8 +8,9 @@ from ...crypto.key import PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKey
|
|||
from ...crypto.key import RepoKey, KeyfileKey, Blake2RepoKey, Blake2KeyfileKey
|
||||
from ...crypto.key import AEADKeyBase
|
||||
from ...crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey
|
||||
from ...crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey
|
||||
from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256
|
||||
from ...crypto.key import Blake3AESOCBRepoKey, Blake3AESOCBKeyfileKey, Blake3CHPORepoKey, Blake3CHPOKeyfileKey
|
||||
from ...crypto.key import Blake3AuthenticatedKey
|
||||
from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, ID_BLAKE3_256
|
||||
from ...crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError
|
||||
from ...crypto.key import identify_key
|
||||
from ...crypto.low_level import IntegrityError as IntegrityErrorBase
|
||||
|
|
@ -76,21 +77,22 @@ class TestKey:
|
|||
# not encrypted
|
||||
PlaintextKey,
|
||||
AuthenticatedKey,
|
||||
Blake2AuthenticatedKey,
|
||||
Blake3AuthenticatedKey,
|
||||
# legacy crypto
|
||||
KeyfileKey,
|
||||
Blake2KeyfileKey,
|
||||
RepoKey,
|
||||
Blake2RepoKey,
|
||||
Blake2AuthenticatedKey,
|
||||
# new crypto
|
||||
AESOCBKeyfileKey,
|
||||
AESOCBRepoKey,
|
||||
Blake2AESOCBKeyfileKey,
|
||||
Blake2AESOCBRepoKey,
|
||||
Blake3AESOCBKeyfileKey,
|
||||
Blake3AESOCBRepoKey,
|
||||
CHPOKeyfileKey,
|
||||
CHPORepoKey,
|
||||
Blake2CHPOKeyfileKey,
|
||||
Blake2CHPORepoKey,
|
||||
Blake3CHPOKeyfileKey,
|
||||
Blake3CHPORepoKey,
|
||||
)
|
||||
)
|
||||
def key(self, request, monkeypatch):
|
||||
|
|
@ -254,6 +256,17 @@ class TestKey:
|
|||
# 0x06 is the key TYPE.
|
||||
assert authenticated == b"\x06" + plaintext
|
||||
|
||||
def test_blake3_authenticated_encrypt(self, monkeypatch):
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
key = Blake3AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
|
||||
assert Blake3AuthenticatedKey.id_hash is ID_BLAKE3_256.id_hash
|
||||
assert len(key.id_key) == 32
|
||||
plaintext = b"123456789"
|
||||
id = key.id_hash(plaintext)
|
||||
authenticated = key.encrypt(id, plaintext)
|
||||
# 0x50 is the key TYPE.
|
||||
assert authenticated == b"\x50" + plaintext
|
||||
|
||||
|
||||
class TestTAM:
|
||||
@pytest.fixture
|
||||
|
|
|
|||
Loading…
Reference in a new issue