upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-03 22:01:05 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Merge pull request #9664 from mr-raj12/legacy-phase6-crypto

Browse source 
legacy: move hardlink helpers, NSIndex1, AES, and PBKDF2 key file methods to borg.legacy, refs #9556
This commit is contained in:
TW 2026-05-26 11:52:54 +02:00 committed by GitHub
commit a461d72525
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 340 additions and 274 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
pyproject.toml
View file

@ -167,6 +167,7 @@ module = [
"pyfuse3",
"trio",
"borg.crypto.low_level",
"borg.legacy.crypto.low_level",
"borg.platform.*",
]
ignore_missing_imports = true

7
setup.py
View file

@ -50,6 +50,7 @@ cflags = ["-Wall", "-Wextra", "-Wpointer-arith", "-Wno-unreachable-code-fallthro
compress_source = "src/borg/compress.pyx"
crypto_ll_source = "src/borg/crypto/low_level.pyx"
crypto_legacy_ll_source = "src/borg/legacy/crypto/low_level.pyx"
buzhash_source = "src/borg/chunkers/buzhash.pyx"
buzhash64_source = "src/borg/chunkers/buzhash64.pyx"
reader_source = "src/borg/chunkers/reader.pyx"
@ -66,6 +67,7 @@ platform_windows_source = "src/borg/platform/windows.pyx"
cython_sources = [
compress_source,
crypto_ll_source,
crypto_legacy_ll_source,
buzhash_source,
buzhash64_source,
reader_source,
@ -155,6 +157,10 @@ if not on_rtd:
dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags)
)
crypto_legacy_ext_kwargs = members_appended(
dict(sources=[crypto_legacy_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags)
)
compress_ext_kwargs = members_appended(
dict(sources=[compress_source]),
lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
@ -174,6 +180,7 @@ if not on_rtd:
ext_modules += [
Extension("borg.crypto.low_level", **crypto_ext_kwargs),
Extension("borg.legacy.crypto.low_level", **crypto_legacy_ext_kwargs),
Extension("borg.compress", **compress_ext_kwargs),
Extension("borg.hashindex", [hashindex_source], extra_compile_args=cflags),
Extension("borg.item", [item_source], extra_compile_args=cflags),

26
src/borg/crypto/key.py
View file

@ -28,7 +28,7 @@ from ..platform import SaveFile
from ..repoobj import RepoObj
from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256
from .low_level import bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256
from .low_level import AES256_OCB, CHACHA20_POLY1305
from . import low_level
@ -438,9 +438,7 @@ class FlexiKey:
raise UnsupportedKeyFormatError()
else:
self._encrypted_key_algorithm = encrypted_key.algorithm
if encrypted_key.algorithm == "sha256":
return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase)
elif encrypted_key.algorithm == "argon2 chacha20-poly1305":
if encrypted_key.algorithm == "argon2 chacha20-poly1305":
return self.decrypt_key_file_argon2(encrypted_key, passphrase)
else:
raise UnsupportedKeyFormatError()
@ -478,13 +476,6 @@ class FlexiKey:
)
return key
def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32)
data = AES(key, b"\0" * 16).decrypt(encrypted_key.data)
if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash):
return data
return None
def decrypt_key_file_argon2(self, encrypted_key, passphrase):
key = self.argon2(
passphrase,
@ -502,22 +493,11 @@ class FlexiKey:
return None
def encrypt_key_file(self, data, passphrase, algorithm):
if algorithm == "sha256":
return self.encrypt_key_file_pbkdf2(data, passphrase)
elif algorithm == "argon2 chacha20-poly1305":
if algorithm == "argon2 chacha20-poly1305":
return self.encrypt_key_file_argon2(data, passphrase)
else:
raise ValueError(f"Unexpected algorithm: {algorithm}")
def encrypt_key_file_pbkdf2(self, data, passphrase):
salt = os.urandom(32)
iterations = PBKDF2_ITERATIONS
key = self.pbkdf2(passphrase, salt, iterations, 32)
hash = hmac_sha256(key, data)
cdata = AES(key, b"\0" * 16).encrypt(data)
enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata)
return msgpack.packb(enc_key.as_dict())
def encrypt_key_file_argon2(self, data, passphrase):
salt = os.urandom(ARGON2_SALT_BYTES)
key = self.argon2(passphrase, output_len_in_bytes=32, salt=salt, **ARGON2_ARGS)

119
src/borg/crypto/low_level.pyx
View file

@ -657,125 +657,6 @@ cdef class CHACHA20_POLY1305(_AEAD_BASE):
super().__init__(key, iv=iv, header_len=header_len, aad_offset=aad_offset)
cdef class AES: # legacy
"""A thin wrapper around the OpenSSL EVP cipher API - for legacy code, like key file encryption"""
cdef CIPHER cipher
cdef EVP_CIPHER_CTX *ctx
cdef unsigned char enc_key[32]
cdef int cipher_blk_len
cdef int iv_len
cdef unsigned char iv[16]
cdef long long blocks
def __init__(self, enc_key, iv=None):
assert isinstance(enc_key, bytes) and len(enc_key) == 32
self.enc_key = enc_key
self.iv_len = 16
assert sizeof(self.iv) == self.iv_len
self.cipher = EVP_aes_256_ctr
self.cipher_blk_len = 16
if iv is not None:
self.set_iv(iv)
else:
self.blocks = -1 # make sure set_iv is called before encrypt
def __cinit__(self, enc_key, iv=None):
self.ctx = EVP_CIPHER_CTX_new()
def __dealloc__(self):
EVP_CIPHER_CTX_free(self.ctx)
def encrypt(self, data, iv=None):
if iv is not None:
self.set_iv(iv)
assert self.blocks == 0, 'iv needs to be set before encrypt is called'
cdef Py_buffer idata
cdef bint idata_acquired = False
cdef unsigned char *odata = NULL
cdef int ilen = len(data)
cdef int olen = 0
cdef int offset
try:
odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
if not odata:
raise MemoryError
idata = ro_buffer(data)
idata_acquired = True
if not EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv):
raise Exception('EVP_EncryptInit_ex failed')
offset = 0
if not EVP_EncryptUpdate(self.ctx, odata, &olen, <const unsigned char*> idata.buf, ilen):
raise Exception('EVP_EncryptUpdate failed')
offset += olen
if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen):
raise Exception('EVP_EncryptFinal failed')
offset += olen
self.blocks = self.block_count(offset)
return odata[:offset]
finally:
if odata:
PyMem_Free(odata)
if idata_acquired:
PyBuffer_Release(&idata)
def decrypt(self, data):
cdef Py_buffer idata
cdef bint idata_acquired = False
cdef unsigned char *odata = NULL
cdef int ilen = len(data)
cdef int offset
cdef int olen = 0
try:
odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
if not odata:
raise MemoryError
idata = ro_buffer(data)
idata_acquired = True
# Set cipher type and mode
if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv):
raise Exception('EVP_DecryptInit_ex failed')
offset = 0
if not EVP_DecryptUpdate(self.ctx, odata, &olen, <const unsigned char*> idata.buf, ilen):
raise Exception('EVP_DecryptUpdate failed')
offset += olen
if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen):
# this error check is very important for modes with padding or
# authentication. for them, a failure here means corrupted data.
# CTR mode does not use padding nor authentication.
raise Exception('EVP_DecryptFinal failed')
offset += olen
self.blocks = self.block_count(ilen)
return odata[:offset]
finally:
if odata:
PyMem_Free(odata)
if idata_acquired:
PyBuffer_Release(&idata)
def block_count(self, length):
return num_cipher_blocks(length, self.cipher_blk_len)
def set_iv(self, iv):
# set_iv needs to be called before each encrypt() call,
# because encrypt does a full initialisation of the cipher context.
if isinstance(iv, int):
iv = iv.to_bytes(self.iv_len, byteorder='big')
assert isinstance(iv, bytes) and len(iv) == self.iv_len
self.iv = iv
self.blocks = 0 # number of cipher blocks encrypted with this IV
def next_iv(self):
# call this after encrypt() to get the next iv (int) for the next encrypt() call
iv = int.from_bytes(self.iv[:self.iv_len], byteorder='big')
return iv + self.blocks
def hmac_sha256(key, data):
return hmac.digest(key, data, 'sha256')

14
src/borg/hashindex.pyi
View file

@ -1,7 +1,9 @@
from typing import NamedTuple, Tuple, Type, IO, Iterator, Any
from typing import NamedTuple, Tuple, Type, IO, Iterator, Any, MutableMapping
PATH_OR_FILE = str | IO
class HTProxyMixin(MutableMapping): ...
class ChunkIndexEntry(NamedTuple):
flags: int
size: int
@ -22,16 +24,6 @@ class ChunkIndex:
def __getitem__(self, key: bytes) -> Type[ChunkIndexEntry]: ...
def __setitem__(self, key: bytes, value: CIE) -> None: ...
class NSIndex1Entry(NamedTuple):
segment: int
offset: int
class NSIndex1: # legacy
def iteritems(self, *args, **kwargs) -> Iterator: ...
def __contains__(self, key: bytes) -> bool: ...
def __getitem__(self, key: bytes) -> Any: ...
def __setitem__(self, key: bytes, value: Any) -> None: ...
class FuseVersionsIndexEntry(NamedTuple):
version: int
hash: bytes

100
src/borg/hashindex.pyx
View file

@ -143,103 +143,3 @@ class FuseVersionsIndex(HTProxyMixin, MutableMapping):
"""
def __init__(self):
self.ht = HashTableNT(key_size=16, value_type=FuseVersionsIndexEntry, value_format=FuseVersionsIndexEntryFormat)
NSIndex1Entry = namedtuple('NSIndex1Entry', 'segment offset')
NSIndex1EntryFormatT = namedtuple('NSIndex1EntryFormatT', 'segment offset')
NSIndex1EntryFormat = NSIndex1EntryFormatT(segment="I", offset="I")
class NSIndex1(HTProxyMixin, MutableMapping):
"""
Mapping from key256 to (segment32, offset32), as used by the legacy repository index of Borg 1.x.
"""
MAX_VALUE = 2**32 - 1 # borghash has the full uint32_t range
MAGIC = b"BORG_IDX" # borg 1.x
HEADER_FMT = "<8sIIBB" # magic, entries, buckets, ksize, vsize
KEY_SIZE = 32
VALUE_SIZE = 8
def __init__(self, capacity=1000, path=None, usable=None):
if usable is not None:
capacity = usable * 2 # load factor 0.5
self.ht = HashTableNT(key_size=self.KEY_SIZE, value_type=NSIndex1Entry, value_format=NSIndex1EntryFormat,
capacity=capacity)
if path:
self._read(path)
def iteritems(self, marker=None):
do_yield = marker is None
for key, value in self.ht.items():
if do_yield:
yield key, value
else:
do_yield = key == marker
@classmethod
def read(cls, path):
return cls(path=path)
def size(self):
return self.ht.size() # not quite correct as this is not the on-disk read-only format.
def write(self, path):
if isinstance(path, str):
with open(path, 'wb') as fd:
self._write_fd(fd)
else:
self._write_fd(path)
def _read(self, path):
if isinstance(path, str):
with open(path, 'rb') as fd:
self._read_fd(fd)
else:
self._read_fd(path)
def _write_fd(self, fd):
used = len(self.ht)
header_bytes = struct.pack(self.HEADER_FMT, self.MAGIC, used, used, self.KEY_SIZE, self.VALUE_SIZE)
fd.write(header_bytes)
# record the header as a separate integrity-hash part if supported
hash_part = getattr(fd, "hash_part", None)
if hash_part:
hash_part("HashHeader")
count = 0
for key, _ in self.ht.items():
value = self.ht._get_raw(key)
fd.write(key)
fd.write(value)
count += 1
assert count == used
def _read_fd(self, fd):
header_size = struct.calcsize(self.HEADER_FMT)
header_bytes = fd.read(header_size)
if len(header_bytes) < header_size:
raise ValueError(f"Invalid file: file is too short (header).")
# verify the header as a separate integrity-hash part if supported
hash_part = getattr(fd, "hash_part", None)
if hash_part:
hash_part("HashHeader")
magic, entries, buckets, ksize, vsize = struct.unpack(self.HEADER_FMT, header_bytes)
if magic != self.MAGIC:
raise ValueError(f"Invalid file: magic {self.MAGIC.decode()} not found.")
assert ksize == self.KEY_SIZE, "invalid key size"
assert vsize == self.VALUE_SIZE, "invalid value size"
buckets_size = buckets * (ksize + vsize)
current_pos = fd.tell()
end_of_file = fd.seek(0, os.SEEK_END)
if current_pos + buckets_size != end_of_file:
raise ValueError(f"Invalid file: file size does not match (buckets).")
fd.seek(current_pos)
for i in range(buckets):
key = fd.read(ksize)
value = fd.read(vsize)
if value.startswith(b'\xFF\xFF\xFF\xFF'): # LE for 0xffffffff (empty/unused bucket)
continue
if value.startswith(b'\xFE\xFF\xFF\xFF'): # LE for 0xfffffffe (deleted/tombstone bucket)
continue
self.ht._set_raw(key, value)
pos = fd.tell()
assert pos == end_of_file

9
src/borg/helpers/fs.py
View file

@ -442,15 +442,6 @@ class HardLinkManager:
self.id_type = id_type
self.info_type = info_type # can be a single type or a tuple of types
def borg1_hardlinkable(self, mode): # legacy
return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode)
def borg1_hardlink_master(self, item): # legacy
return item.get("hardlink_master", False) and "source" not in item and self.borg1_hardlinkable(item.mode)
def borg1_hardlink_slave(self, item): # legacy
return "source" in item and self.borg1_hardlinkable(item.mode)
def hardlink_id_from_path(self, path):
"""compute a hard link id from a path"""
assert isinstance(path, str)

55
src/borg/legacy/crypto/key.py
View file

@ -1,9 +1,52 @@
import hmac
import os
from ...constants import * # NOQA
from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b
from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, AESKeyBase, FlexiKey
from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, hmac_sha256
from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, AESKeyBase, FlexiKey, UnsupportedKeyFormatError
from ...helpers import get_limited_unpacker, msgpack
from ...item import EncryptedKey
from .low_level import AES
class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
class Pbkdf2FileMixin:
"""Mixin for borg 1.x key files encrypted with PBKDF2 + AES-CTR."""
def decrypt_key_file(self, data, passphrase):
unpacker = get_limited_unpacker("key")
unpacker.feed(data)
unpacked = unpacker.unpack()
encrypted_key = EncryptedKey(internal_dict=unpacked)
if encrypted_key.version != 1:
raise UnsupportedKeyFormatError()
self._encrypted_key_algorithm = encrypted_key.algorithm
if encrypted_key.algorithm == "sha256":
return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase)
return super().decrypt_key_file(data, passphrase)
def encrypt_key_file(self, data, passphrase, algorithm):
if algorithm == "sha256":
return self.encrypt_key_file_pbkdf2(data, passphrase)
return super().encrypt_key_file(data, passphrase, algorithm)
def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32)
data = AES(key, b"\0" * 16).decrypt(encrypted_key.data)
if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash):
return data
return None
def encrypt_key_file_pbkdf2(self, data, passphrase):
salt = os.urandom(32)
iterations = PBKDF2_ITERATIONS
key = self.pbkdf2(passphrase, salt, iterations, 32)
hash = hmac_sha256(key, data)
cdata = AES(key, b"\0" * 16).encrypt(data)
enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata)
return msgpack.packb(enc_key.as_dict())
class KeyfileKey(Pbkdf2FileMixin, ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE}
TYPE = KeyType.KEYFILE
NAME = "key file"
@ -12,7 +55,7 @@ class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
CIPHERSUITE = AES256_CTR_HMAC_SHA256
class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
class RepoKey(Pbkdf2FileMixin, ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE}
TYPE = KeyType.REPO
NAME = "repokey"
@ -21,7 +64,7 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc]
CIPHERSUITE = AES256_CTR_HMAC_SHA256
class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc]
class Blake2KeyfileKey(Pbkdf2FileMixin, ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc]
TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
TYPE = KeyType.BLAKE2KEYFILE
NAME = "key file BLAKE2b"
@ -30,7 +73,7 @@ class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[mi
CIPHERSUITE = AES256_CTR_BLAKE2b
class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc]
class Blake2RepoKey(Pbkdf2FileMixin, ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc]
TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
TYPE = KeyType.BLAKE2REPO
NAME = "repokey BLAKE2b"

147
src/borg/legacy/crypto/low_level.pyx Normal file
View file

@ -0,0 +1,147 @@
from cpython cimport PyMem_Malloc, PyMem_Free
from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
cdef extern from "openssl/evp.h":
ctypedef struct EVP_CIPHER:
pass
ctypedef struct EVP_CIPHER_CTX:
pass
ctypedef struct ENGINE:
pass
const EVP_CIPHER *EVP_aes_256_ctr()
EVP_CIPHER_CTX *EVP_CIPHER_CTX_new()
void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a)
int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
const unsigned char *key, const unsigned char *iv)
int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
const unsigned char *key, const unsigned char *iv)
int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in_, int inl)
int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in_, int inl)
int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
ctypedef const EVP_CIPHER * (* CIPHER)()
cdef Py_buffer ro_buffer(object data) except *:
cdef Py_buffer view
PyObject_GetBuffer(data, &view, PyBUF_SIMPLE)
return view
cdef class AES:
"""A thin wrapper around the OpenSSL EVP cipher API - for legacy key file encryption."""
cdef CIPHER cipher
cdef EVP_CIPHER_CTX *ctx
cdef unsigned char enc_key[32]
cdef int cipher_blk_len
cdef int iv_len
cdef unsigned char iv[16]
cdef long long blocks
def __init__(self, enc_key, iv=None):
assert isinstance(enc_key, bytes) and len(enc_key) == 32
self.enc_key = enc_key
self.iv_len = 16
assert sizeof(self.iv) == self.iv_len
self.cipher = EVP_aes_256_ctr
self.cipher_blk_len = 16
if iv is not None:
self.set_iv(iv)
else:
self.blocks = -1 # make sure set_iv is called before encrypt
def __cinit__(self, enc_key, iv=None):
self.ctx = EVP_CIPHER_CTX_new()
def __dealloc__(self):
EVP_CIPHER_CTX_free(self.ctx)
def encrypt(self, data, iv=None):
if iv is not None:
self.set_iv(iv)
assert self.blocks == 0, 'iv needs to be set before encrypt is called'
cdef Py_buffer idata
cdef bint idata_acquired = False
cdef unsigned char *odata = NULL
cdef int ilen = len(data)
cdef int olen = 0
cdef int offset
try:
odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
if not odata:
raise MemoryError
idata = ro_buffer(data)
idata_acquired = True
if not EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv):
raise Exception('EVP_EncryptInit_ex failed')
offset = 0
if not EVP_EncryptUpdate(self.ctx, odata, &olen, <const unsigned char*> idata.buf, ilen):
raise Exception('EVP_EncryptUpdate failed')
offset += olen
if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen):
raise Exception('EVP_EncryptFinal failed')
offset += olen
self.blocks = self.block_count(offset)
return odata[:offset]
finally:
if odata:
PyMem_Free(odata)
if idata_acquired:
PyBuffer_Release(&idata)
def decrypt(self, data):
cdef Py_buffer idata
cdef bint idata_acquired = False
cdef unsigned char *odata = NULL
cdef int ilen = len(data)
cdef int offset
cdef int olen = 0
try:
odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
if not odata:
raise MemoryError
idata = ro_buffer(data)
idata_acquired = True
if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv):
raise Exception('EVP_DecryptInit_ex failed')
offset = 0
if not EVP_DecryptUpdate(self.ctx, odata, &olen, <const unsigned char*> idata.buf, ilen):
raise Exception('EVP_DecryptUpdate failed')
offset += olen
if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen):
raise Exception('EVP_DecryptFinal failed')
offset += olen
self.blocks = self.block_count(ilen)
return odata[:offset]
finally:
if odata:
PyMem_Free(odata)
if idata_acquired:
PyBuffer_Release(&idata)
def block_count(self, length):
return (length + self.cipher_blk_len - 1) // self.cipher_blk_len
def set_iv(self, iv):
if isinstance(iv, int):
iv = iv.to_bytes(self.iv_len, byteorder='big')
assert isinstance(iv, bytes) and len(iv) == self.iv_len
self.iv = iv
self.blocks = 0
def next_iv(self):
iv = int.from_bytes(self.iv[:self.iv_len], byteorder='big')
return iv + self.blocks

108
src/borg/legacy/hashindex.py Normal file
View file

@ -0,0 +1,108 @@
from collections.abc import MutableMapping
from collections import namedtuple
import os
import struct
from borghash import HashTableNT
from ..hashindex import HTProxyMixin
NSIndex1Entry = namedtuple("NSIndex1Entry", "segment offset")
NSIndex1EntryFormatT = namedtuple("NSIndex1EntryFormatT", "segment offset")
NSIndex1EntryFormat = NSIndex1EntryFormatT(segment="I", offset="I")
class NSIndex1(HTProxyMixin, MutableMapping):
"""
Mapping from key256 to (segment32, offset32), as used by the legacy repository index of Borg 1.x.
"""
MAX_VALUE = 2**32 - 1 # borghash has the full uint32_t range
MAGIC = b"BORG_IDX" # borg 1.x
HEADER_FMT = "<8sIIBB" # magic, entries, buckets, ksize, vsize
KEY_SIZE = 32
VALUE_SIZE = 8
def __init__(self, capacity=1000, path=None, usable=None):
if usable is not None:
capacity = usable * 2 # load factor 0.5
self.ht = HashTableNT(
key_size=self.KEY_SIZE, value_type=NSIndex1Entry, value_format=NSIndex1EntryFormat, capacity=capacity
)
if path:
self._read(path)
def iteritems(self, marker=None):
do_yield = marker is None
for key, value in self.ht.items():
if do_yield:
yield key, value
else:
do_yield = key == marker
@classmethod
def read(cls, path):
return cls(path=path)
def size(self):
return self.ht.size() # not quite correct as this is not the on-disk read-only format.
def write(self, path):
if isinstance(path, str):
with open(path, "wb") as fd:
self._write_fd(fd)
else:
self._write_fd(path)
def _read(self, path):
if isinstance(path, str):
with open(path, "rb") as fd:
self._read_fd(fd)
else:
self._read_fd(path)
def _write_fd(self, fd):
used = len(self.ht)
header_bytes = struct.pack(self.HEADER_FMT, self.MAGIC, used, used, self.KEY_SIZE, self.VALUE_SIZE)
fd.write(header_bytes)
hash_part = getattr(fd, "hash_part", None)
if hash_part:
hash_part("HashHeader")
count = 0
for key, _ in self.ht.items():
value = self.ht._get_raw(key)
fd.write(key)
fd.write(value)
count += 1
assert count == used
def _read_fd(self, fd):
header_size = struct.calcsize(self.HEADER_FMT)
header_bytes = fd.read(header_size)
if len(header_bytes) < header_size:
raise ValueError("Invalid file: file is too short (header).")
hash_part = getattr(fd, "hash_part", None)
if hash_part:
hash_part("HashHeader")
magic, entries, buckets, ksize, vsize = struct.unpack(self.HEADER_FMT, header_bytes)
if magic != self.MAGIC:
raise ValueError(f"Invalid file: magic {self.MAGIC.decode()} not found.")
assert ksize == self.KEY_SIZE, "invalid key size"
assert vsize == self.VALUE_SIZE, "invalid value size"
buckets_size = buckets * (ksize + vsize)
current_pos = fd.tell()
end_of_file = fd.seek(0, os.SEEK_END)
if current_pos + buckets_size != end_of_file:
raise ValueError("Invalid file: file size does not match (buckets).")
fd.seek(current_pos)
for i in range(buckets):
key = fd.read(ksize)
value = fd.read(vsize)
if value.startswith(b"\xff\xff\xff\xff"): # LE for 0xffffffff (empty/unused bucket)
continue
if value.startswith(b"\xfe\xff\xff\xff"): # LE for 0xfffffffe (deleted/tombstone bucket)
continue
self.ht._set_raw(key, value)
pos = fd.tell()
assert pos == end_of_file

13
src/borg/legacy/helpers.py Normal file
View file

@ -0,0 +1,13 @@
import stat
def borg1_hardlinkable(mode):
return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode)
def borg1_hardlink_master(item):
return item.get("hardlink_master", False) and "source" not in item and borg1_hardlinkable(item.mode)
def borg1_hardlink_slave(item):
return "source" in item and borg1_hardlinkable(item.mode)

2
src/borg/legacy/repository.py
View file

@ -16,7 +16,7 @@ from zlib import crc32
import xxhash
from ..constants import * # NOQA
from ..hashindex import NSIndex1Entry, NSIndex1
from .hashindex import NSIndex1Entry, NSIndex1
from ..helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size
from ..helpers import Location
from ..helpers import ProgressIndicatorPercent

5
src/borg/legacy/upgrade.py
View file

@ -6,6 +6,7 @@ from ..compress import ZLIB, ZLIB_legacy, ObfuscateSize
from ..helpers import HardLinkManager, join_cmd
from ..item import Item
from ..logger import create_logger
from .helpers import borg1_hardlink_master, borg1_hardlink_slave
logger = create_logger(__name__)
@ -47,10 +48,10 @@ class UpgraderFrom12To20:
"acl_extended",
}
if self.hlm.borg1_hardlink_master(item):
if borg1_hardlink_master(item):
item.hlid = hlid = self.hlm.hardlink_id_from_path(item.path)
self.hlm.remember(id=hlid, info=item.get("chunks"))
elif self.hlm.borg1_hardlink_slave(item):
elif borg1_hardlink_slave(item):
item.hlid = hlid = self.hlm.hardlink_id_from_path(item.source)
chunks = self.hlm.retrieve(id=hlid)
if chunks is not None:

6
src/borg/testsuite/crypto/crypto_test.py
View file

@ -6,9 +6,11 @@ import unittest
from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, IntegrityError
from ...crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes
from ...crypto.low_level import AES, hmac_sha256
from ...crypto.low_level import hmac_sha256
from ...legacy.crypto.low_level import AES
from hashlib import sha256
from ...crypto.key import CHPOKeyfileKey, AESOCBRepoKey, FlexiKey, KeyBase, PlaintextKey
from ...legacy.crypto.key import KeyfileKey as LegacyKeyfileKey
from ...helpers import msgpack, bin_to_hex
from .. import BaseTestCase
@ -232,7 +234,7 @@ def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256():
encrypted = msgpack.packb(
{"version": 1, "algorithm": "sha256", "iterations": 1, "salt": salt, "data": data, "hash": hash}
)
key = CHPOKeyfileKey(None)
key = LegacyKeyfileKey(None)
decrypted = key.decrypt_key_file(encrypted, passphrase)

2
src/borg/testsuite/legacyrepository_test.py
View file

@ -7,7 +7,7 @@ from unittest.mock import patch
import pytest
from xxhash import xxh64
from ..hashindex import NSIndex1
from ..legacy.hashindex import NSIndex1
from ..helpers import Location
from ..helpers import IntegrityError
from ..helpers import msgpack