From 0306ba9a63f01248e18c5eb7b67a48fef3adcae1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 13 Aug 2024 22:39:43 +0200 Subject: [PATCH] get rid of the CacheSynchronizer Lots of low-level code written back then to optimize runtime of some functions. We'll solve this differently by doing less stats, esp. if it is expensive to compute. --- scripts/fuzz-cache-sync/HOWTO | 10 - scripts/fuzz-cache-sync/main.c | 33 -- .../fuzz-cache-sync/testcase_dir/test_simple | Bin 147 -> 0 bytes src/borg/cache_sync/cache_sync.c | 138 ------ src/borg/cache_sync/sysdep.h | 194 -------- src/borg/cache_sync/unpack.h | 416 ------------------ src/borg/cache_sync/unpack_define.h | 95 ---- src/borg/cache_sync/unpack_template.h | 365 --------------- src/borg/hashindex.pyx | 53 +-- src/borg/testsuite/cache.py | 147 +------ 10 files changed, 2 insertions(+), 1449 deletions(-) delete mode 100644 scripts/fuzz-cache-sync/HOWTO delete mode 100644 scripts/fuzz-cache-sync/main.c delete mode 100644 scripts/fuzz-cache-sync/testcase_dir/test_simple delete mode 100644 src/borg/cache_sync/cache_sync.c delete mode 100644 src/borg/cache_sync/sysdep.h delete mode 100644 src/borg/cache_sync/unpack.h delete mode 100644 src/borg/cache_sync/unpack_define.h delete mode 100644 src/borg/cache_sync/unpack_template.h diff --git a/scripts/fuzz-cache-sync/HOWTO b/scripts/fuzz-cache-sync/HOWTO deleted file mode 100644 index ae144b287..000000000 --- a/scripts/fuzz-cache-sync/HOWTO +++ /dev/null @@ -1,10 +0,0 @@ -- Install AFL and the requirements for LLVM mode (see docs) -- Compile the fuzzing target, e.g. - - AFL_HARDEN=1 afl-clang-fast main.c -o fuzz-target -O3 - - (other options, like using ASan or MSan are possible as well) -- Add additional test cases to testcase_dir -- Run afl, easiest (but inefficient) way; - - afl-fuzz -i testcase_dir -o findings_dir ./fuzz-target diff --git a/scripts/fuzz-cache-sync/main.c b/scripts/fuzz-cache-sync/main.c deleted file mode 100644 index c65dd272d..000000000 --- a/scripts/fuzz-cache-sync/main.c +++ /dev/null @@ -1,33 +0,0 @@ - -#define BORG_NO_PYTHON - -#include "../../src/borg/_hashindex.c" -#include "../../src/borg/cache_sync/cache_sync.c" - -#define BUFSZ 32768 - -int main() { - char buf[BUFSZ]; - int len, ret; - CacheSyncCtx *ctx; - HashIndex *idx; - - /* capacity, key size, value size */ - idx = hashindex_init(0, 32, 12); - ctx = cache_sync_init(idx); - - while (1) { - len = read(0, buf, BUFSZ); - if (!len) { - break; - } - ret = cache_sync_feed(ctx, buf, len); - if(!ret && cache_sync_error(ctx)) { - fprintf(stderr, "error: %s\n", cache_sync_error(ctx)); - return 1; - } - } - hashindex_free(idx); - cache_sync_free(ctx); - return 0; -} diff --git a/scripts/fuzz-cache-sync/testcase_dir/test_simple b/scripts/fuzz-cache-sync/testcase_dir/test_simple deleted file mode 100644 index d5f6670c154abeb45b981330307e6f4ac7e5e3ee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147 zcmaFAfA9PKd(-msfn-u*5tyt3k}Q`NL%3pkKwfEaDo83hqcktO7?}ZN0}+FQ0e)b} L$V8+{BPM15ZpAyL diff --git a/src/borg/cache_sync/cache_sync.c b/src/borg/cache_sync/cache_sync.c deleted file mode 100644 index eff765d49..000000000 --- a/src/borg/cache_sync/cache_sync.c +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Borg cache synchronizer, - * high level interface. - * - * These routines parse msgpacked item metadata and update a HashIndex - * with all chunks that are referenced from the items. - * - * This file only contains some initialization and buffer management. - * - * The parser is split in two parts, somewhat similar to lexer/parser combinations: - * - * unpack_template.h munches msgpack and calls a specific callback for each object - * encountered (e.g. beginning of a map, an integer, a string, a map item etc.). - * - * unpack.h implements these callbacks and uses another state machine to - * extract chunk references from it. - */ - -#include "unpack.h" - -typedef struct { - unpack_context ctx; - - char *buf; - size_t head; - size_t tail; - size_t size; -} CacheSyncCtx; - -static CacheSyncCtx * -cache_sync_init(HashIndex *chunks) -{ - CacheSyncCtx *ctx; - if (!(ctx = (CacheSyncCtx*)malloc(sizeof(CacheSyncCtx)))) { - return NULL; - } - - unpack_init(&ctx->ctx); - /* needs to be set only once */ - ctx->ctx.user.chunks = chunks; - ctx->ctx.user.totals.size = 0; - ctx->ctx.user.totals.num_files = 0; - ctx->buf = NULL; - ctx->head = 0; - ctx->tail = 0; - ctx->size = 0; - - return ctx; -} - -static void -cache_sync_free(CacheSyncCtx *ctx) -{ - if(ctx->buf) { - free(ctx->buf); - } - free(ctx); -} - -static const char * -cache_sync_error(const CacheSyncCtx *ctx) -{ - return ctx->ctx.user.last_error; -} - -static uint64_t -cache_sync_num_files_totals(const CacheSyncCtx *ctx) -{ - return ctx->ctx.user.totals.num_files; -} - -static uint64_t -cache_sync_size_totals(const CacheSyncCtx *ctx) -{ - return ctx->ctx.user.totals.size; -} - -/** - * feed data to the cache synchronizer - * 0 = abort, 1 = continue - * abort is a regular condition, check cache_sync_error - */ -static int -cache_sync_feed(CacheSyncCtx *ctx, void *data, uint32_t length) -{ - size_t new_size; - int ret; - char *new_buf; - - if(ctx->tail + length > ctx->size) { - if((ctx->tail - ctx->head) + length <= ctx->size) { - /* | XXXXX| -> move data in buffer backwards -> |XXXXX | */ - memmove(ctx->buf, ctx->buf + ctx->head, ctx->tail - ctx->head); - ctx->tail -= ctx->head; - ctx->head = 0; - } else { - /* must expand buffer to fit all data */ - new_size = (ctx->tail - ctx->head) + length; - new_buf = (char*) malloc(new_size); - if(!new_buf) { - ctx->ctx.user.last_error = "cache_sync_feed: unable to allocate buffer"; - return 0; - } - if(ctx->buf) { - memcpy(new_buf, ctx->buf + ctx->head, ctx->tail - ctx->head); - free(ctx->buf); - } - ctx->buf = new_buf; - ctx->tail -= ctx->head; - ctx->head = 0; - ctx->size = new_size; - } - } - - memcpy(ctx->buf + ctx->tail, data, length); - ctx->tail += length; - - while(1) { - if(ctx->head >= ctx->tail) { - return 1; /* request more bytes */ - } - - ret = unpack_execute(&ctx->ctx, ctx->buf, ctx->tail, &ctx->head); - if(ret == 1) { - unpack_init(&ctx->ctx); - continue; - } else if(ret == 0) { - return 1; - } else { - if(!ctx->ctx.user.last_error) { - ctx->ctx.user.last_error = "Unknown error"; - } - return 0; - } - } - /* unreachable */ - return 1; -} diff --git a/src/borg/cache_sync/sysdep.h b/src/borg/cache_sync/sysdep.h deleted file mode 100644 index e4ce7850f..000000000 --- a/src/borg/cache_sync/sysdep.h +++ /dev/null @@ -1,194 +0,0 @@ -/* - * MessagePack system dependencies - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_SYSDEP_H__ -#define MSGPACK_SYSDEP_H__ - -#include -#include -#if defined(_MSC_VER) && _MSC_VER < 1600 -typedef __int8 int8_t; -typedef unsigned __int8 uint8_t; -typedef __int16 int16_t; -typedef unsigned __int16 uint16_t; -typedef __int32 int32_t; -typedef unsigned __int32 uint32_t; -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -#elif defined(_MSC_VER) // && _MSC_VER >= 1600 -#include -#else -#include -#include -#endif - -#ifdef _WIN32 -#define _msgpack_atomic_counter_header -typedef long _msgpack_atomic_counter_t; -#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) -#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) -#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41) -#define _msgpack_atomic_counter_header "gcc_atomic.h" -#else -typedef unsigned int _msgpack_atomic_counter_t; -#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) -#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) -#endif - -#ifdef _WIN32 - -#ifdef __cplusplus -/* numeric_limits::min,max */ -#ifdef max -#undef max -#endif -#ifdef min -#undef min -#endif -#endif - -#else -#include /* __BYTE_ORDER */ -#endif - -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#elif _WIN32 -#define __LITTLE_ENDIAN__ -#endif -#endif - - -#ifdef __LITTLE_ENDIAN__ - -#ifdef _WIN32 -# if defined(ntohs) -# define _msgpack_be16(x) ntohs(x) -# elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400) -# define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x)) -# else -# define _msgpack_be16(x) ( \ - ((((uint16_t)x) << 8) ) | \ - ((((uint16_t)x) >> 8) ) ) -# endif -#else -# define _msgpack_be16(x) ntohs(x) -#endif - -#ifdef _WIN32 -# if defined(ntohl) -# define _msgpack_be32(x) ntohl(x) -# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400) -# define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x)) -# else -# define _msgpack_be32(x) \ - ( ((((uint32_t)x) << 24) ) | \ - ((((uint32_t)x) << 8) & 0x00ff0000U ) | \ - ((((uint32_t)x) >> 8) & 0x0000ff00U ) | \ - ((((uint32_t)x) >> 24) ) ) -# endif -#else -# define _msgpack_be32(x) ntohl(x) -#endif - -#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400) -# define _msgpack_be64(x) (_byteswap_uint64(x)) -#elif defined(bswap_64) -# define _msgpack_be64(x) bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) -#else -#define _msgpack_be64(x) \ - ( ((((uint64_t)x) << 56) ) | \ - ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ - ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ - ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ - ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ - ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ - ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ - ((((uint64_t)x) >> 56) ) ) -#endif - -#define _msgpack_load16(cast, from) ((cast)( \ - (((uint16_t)((uint8_t*)(from))[0]) << 8) | \ - (((uint16_t)((uint8_t*)(from))[1]) ) )) - -#define _msgpack_load32(cast, from) ((cast)( \ - (((uint32_t)((uint8_t*)(from))[0]) << 24) | \ - (((uint32_t)((uint8_t*)(from))[1]) << 16) | \ - (((uint32_t)((uint8_t*)(from))[2]) << 8) | \ - (((uint32_t)((uint8_t*)(from))[3]) ) )) - -#define _msgpack_load64(cast, from) ((cast)( \ - (((uint64_t)((uint8_t*)(from))[0]) << 56) | \ - (((uint64_t)((uint8_t*)(from))[1]) << 48) | \ - (((uint64_t)((uint8_t*)(from))[2]) << 40) | \ - (((uint64_t)((uint8_t*)(from))[3]) << 32) | \ - (((uint64_t)((uint8_t*)(from))[4]) << 24) | \ - (((uint64_t)((uint8_t*)(from))[5]) << 16) | \ - (((uint64_t)((uint8_t*)(from))[6]) << 8) | \ - (((uint64_t)((uint8_t*)(from))[7]) ) )) - -#else - -#define _msgpack_be16(x) (x) -#define _msgpack_be32(x) (x) -#define _msgpack_be64(x) (x) - -#define _msgpack_load16(cast, from) ((cast)( \ - (((uint16_t)((uint8_t*)from)[0]) << 8) | \ - (((uint16_t)((uint8_t*)from)[1]) ) )) - -#define _msgpack_load32(cast, from) ((cast)( \ - (((uint32_t)((uint8_t*)from)[0]) << 24) | \ - (((uint32_t)((uint8_t*)from)[1]) << 16) | \ - (((uint32_t)((uint8_t*)from)[2]) << 8) | \ - (((uint32_t)((uint8_t*)from)[3]) ) )) - -#define _msgpack_load64(cast, from) ((cast)( \ - (((uint64_t)((uint8_t*)from)[0]) << 56) | \ - (((uint64_t)((uint8_t*)from)[1]) << 48) | \ - (((uint64_t)((uint8_t*)from)[2]) << 40) | \ - (((uint64_t)((uint8_t*)from)[3]) << 32) | \ - (((uint64_t)((uint8_t*)from)[4]) << 24) | \ - (((uint64_t)((uint8_t*)from)[5]) << 16) | \ - (((uint64_t)((uint8_t*)from)[6]) << 8) | \ - (((uint64_t)((uint8_t*)from)[7]) ) )) -#endif - - -#define _msgpack_store16(to, num) \ - do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0) -#define _msgpack_store32(to, num) \ - do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0) -#define _msgpack_store64(to, num) \ - do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0) - -/* -#define _msgpack_load16(cast, from) \ - ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); }) -#define _msgpack_load32(cast, from) \ - ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); }) -#define _msgpack_load64(cast, from) \ - ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); }) -*/ - - -#endif /* msgpack/sysdep.h */ diff --git a/src/borg/cache_sync/unpack.h b/src/borg/cache_sync/unpack.h deleted file mode 100644 index ad898655f..000000000 --- a/src/borg/cache_sync/unpack.h +++ /dev/null @@ -1,416 +0,0 @@ -/* - * Borg cache synchronizer, - * based on a MessagePack for Python unpacking routine - * - * Copyright (C) 2009 Naoki INADA - * Copyright (c) 2017 Marian Beermann - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This limits the depth of the structures we can unpack, i.e. how many containers - * are nestable. - */ -#define MSGPACK_EMBED_STACK_SIZE (16) -#include "unpack_define.h" - -// 2**32 - 1025 -#define _MAX_VALUE ( (uint32_t) 4294966271UL ) - -#define MIN(x, y) ((x) < (y) ? (x): (y)) - -#ifdef DEBUG -#define SET_LAST_ERROR(msg) \ - fprintf(stderr, "cache_sync parse error: %s\n", (msg)); \ - u->last_error = (msg); -#else -#define SET_LAST_ERROR(msg) \ - u->last_error = (msg); -#endif - -typedef struct unpack_user { - /* Item.chunks is at the top level; we don't care about anything else, - * only need to track the current level to navigate arbitrary and unknown structure. - * To discern keys from everything else on the top level we use expect_map_item_end. - */ - int level; - - const char *last_error; - - HashIndex *chunks; - - /* - * We don't care about most stuff. This flag tells us whether we're at the chunks structure, - * meaning: - * {'foo': 'bar', 'chunks': [...], 'stuff': ... } - * ^-HERE-^ - */ - int inside_chunks; - - /* does this item have a chunks list in it? */ - int has_chunks; - - enum { - /* the next thing is a map key at the Item root level, - * and it might be e.g. the "chunks" key we're looking for */ - expect_map_key, - - /* blocking state to expect_map_key - * { 'stuff': , 'chunks': [ - * emk -> emie -> -> -> -> emk ecb eeboce - * (nested containers are tracked via level) - * emk=expect_map_key, emie=expect_map_item_end, ecb=expect_chunks_begin, - * eeboce=expect_entry_begin_or_chunks_end - */ - expect_map_item_end, - - /* next thing must be the chunks array (array) */ - expect_chunks_begin, - - /* next thing must either be another CLE (array) or end of Item.chunks (array_end) */ - expect_entry_begin_or_chunks_end, - - /* - * processing ChunkListEntry tuple: - * expect_key, expect_size, expect_entry_end - */ - /* next thing must be the key (raw, l=32) */ - expect_key, - /* next thing must be the size (int) */ - expect_size, - /* next thing must be the end of the CLE (array_end) */ - expect_entry_end, - - expect_item_begin - } expect; - - /* collect values here for current chunklist entry */ - struct { - unsigned char key[32]; - uint32_t size; - } current; - - /* summing up chunks sizes here within a single item */ - struct { - uint64_t size; - } item; - - /* total sizes and files count coming from all files */ - struct { - uint64_t size, num_files; - } totals; - -} unpack_user; - -struct unpack_context; -typedef struct unpack_context unpack_context; -typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off); - -#define UNEXPECTED(what) \ - if(u->inside_chunks || u->expect == expect_map_key) { \ - SET_LAST_ERROR("Unexpected object: " what); \ - return -1; \ - } - -static inline void unpack_init_user_state(unpack_user *u) -{ - u->last_error = NULL; - u->level = 0; - u->inside_chunks = 0; - u->expect = expect_item_begin; -} - -static inline int unpack_callback_uint64(unpack_user* u, int64_t d) -{ - switch(u->expect) { - case expect_size: - u->current.size = d; - u->expect = expect_entry_end; - break; - default: - UNEXPECTED("integer"); - } - return 0; -} - -static inline int unpack_callback_uint32(unpack_user* u, uint32_t d) -{ - return unpack_callback_uint64(u, d); -} - -static inline int unpack_callback_uint16(unpack_user* u, uint16_t d) -{ - return unpack_callback_uint64(u, d); -} - -static inline int unpack_callback_uint8(unpack_user* u, uint8_t d) -{ - return unpack_callback_uint64(u, d); -} - -static inline int unpack_callback_int64(unpack_user* u, uint64_t d) -{ - return unpack_callback_uint64(u, d); -} - -static inline int unpack_callback_int32(unpack_user* u, int32_t d) -{ - return unpack_callback_uint64(u, d); -} - -static inline int unpack_callback_int16(unpack_user* u, int16_t d) -{ - return unpack_callback_uint64(u, d); -} - -static inline int unpack_callback_int8(unpack_user* u, int8_t d) -{ - return unpack_callback_uint64(u, d); -} - -/* Ain't got anything to do with those floats */ -static inline int unpack_callback_double(unpack_user* u, double d) -{ - (void)d; - UNEXPECTED("double"); - return 0; -} - -static inline int unpack_callback_float(unpack_user* u, float d) -{ - (void)d; - UNEXPECTED("float"); - return 0; -} - -/* nil/true/false — I/don't/care */ -static inline int unpack_callback_nil(unpack_user* u) -{ - UNEXPECTED("nil"); - return 0; -} - -static inline int unpack_callback_true(unpack_user* u) -{ - UNEXPECTED("true"); - return 0; -} - -static inline int unpack_callback_false(unpack_user* u) -{ - UNEXPECTED("false"); - return 0; -} - -static inline int unpack_callback_array(unpack_user* u, unsigned int n) -{ - switch(u->expect) { - case expect_chunks_begin: - /* b'chunks': [ - * ^ */ - u->expect = expect_entry_begin_or_chunks_end; - break; - case expect_entry_begin_or_chunks_end: - /* b'chunks': [ ( - * ^ */ - if(n != 2) { - SET_LAST_ERROR("Invalid chunk list entry length"); - return -1; - } - u->expect = expect_key; - break; - default: - if(u->inside_chunks) { - SET_LAST_ERROR("Unexpected array start"); - return -1; - } else { - u->level++; - return 0; - } - } - return 0; -} - -static inline int unpack_callback_array_item(unpack_user* u, unsigned int current) -{ - (void)u; (void)current; - return 0; -} - -static inline int unpack_callback_array_end(unpack_user* u) -{ - uint32_t *cache_entry; - uint32_t cache_values[3]; - uint64_t refcount; - - switch(u->expect) { - case expect_entry_end: - /* b'chunks': [ ( b'1234...', 123, 345 ) - * ^ */ - cache_entry = (uint32_t*) hashindex_get(u->chunks, u->current.key); - if(cache_entry) { - refcount = _le32toh(cache_entry[0]); - if(refcount > _MAX_VALUE) { - SET_LAST_ERROR("invalid reference count"); - return -1; - } - refcount += 1; - cache_entry[0] = _htole32(MIN(refcount, _MAX_VALUE)); - } else { - /* refcount, size */ - cache_values[0] = _htole32(1); - cache_values[1] = _htole32(u->current.size); - if(!hashindex_set(u->chunks, u->current.key, cache_values)) { - SET_LAST_ERROR("hashindex_set failed"); - return -1; - } - } - u->item.size += u->current.size; - u->expect = expect_entry_begin_or_chunks_end; - break; - case expect_entry_begin_or_chunks_end: - /* b'chunks': [ ] - * ^ */ - /* end of Item.chunks */ - u->inside_chunks = 0; - u->expect = expect_map_item_end; - break; - default: - if(u->inside_chunks) { - SET_LAST_ERROR("Invalid state transition (unexpected array end)"); - return -1; - } else { - u->level--; - return 0; - } - } - return 0; -} - -static inline int unpack_callback_map(unpack_user* u, unsigned int n) -{ - (void)n; - - if(u->level == 0) { - if(u->expect != expect_item_begin) { - SET_LAST_ERROR("Invalid state transition"); /* unreachable */ - return -1; - } - /* This begins a new Item */ - u->expect = expect_map_key; - u->has_chunks = 0; - u->item.size = 0; - } - - if(u->inside_chunks) { - UNEXPECTED("map"); - } - - u->level++; - - return 0; -} - -static inline int unpack_callback_map_item(unpack_user* u, unsigned int current) -{ - (void)u; (void)current; - - if(u->level == 1) { - switch(u->expect) { - case expect_map_item_end: - u->expect = expect_map_key; - break; - default: - SET_LAST_ERROR("Unexpected map item"); - return -1; - } - } - return 0; -} - -static inline int unpack_callback_map_end(unpack_user* u) -{ - u->level--; - if(u->inside_chunks) { - SET_LAST_ERROR("Unexpected map end"); - return -1; - } - if(u->level == 0) { - /* This ends processing of an Item */ - if(u->has_chunks) { - u->totals.num_files += 1; - u->totals.size += u->item.size; - } - } - return 0; -} - -static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int length) -{ - /* raw = what Borg uses for text stuff */ - /* Note: p points to an internal buffer which contains l bytes. */ - (void)b; - - switch(u->expect) { - case expect_map_key: - if(length == 6 && !memcmp("chunks", p, 6)) { - u->expect = expect_chunks_begin; - u->inside_chunks = 1; - u->has_chunks = 1; - } else { - u->expect = expect_map_item_end; - } - break; - default: - if(u->inside_chunks) { - SET_LAST_ERROR("Unexpected raw in chunks structure"); - return -1; - } - } - return 0; -} - -static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int length) -{ - /* bin = what Borg uses for binary stuff */ - /* Note: p points to an internal buffer which contains l bytes. */ - (void)b; - - switch(u->expect) { - case expect_key: - if(length != 32) { - SET_LAST_ERROR("Incorrect key length"); - return -1; - } - memcpy(u->current.key, p, 32); - u->expect = expect_size; - break; - default: - if(u->inside_chunks) { - SET_LAST_ERROR("Unexpected bytes in chunks structure"); - return -1; - } - } - return 0; -} - -static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, - unsigned int length) -{ - (void)u; (void)base; (void)pos; (void)length; - UNEXPECTED("ext"); - return 0; -} - -#include "unpack_template.h" diff --git a/src/borg/cache_sync/unpack_define.h b/src/borg/cache_sync/unpack_define.h deleted file mode 100644 index 10c910861..000000000 --- a/src/borg/cache_sync/unpack_define.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * MessagePack unpacking routine template - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_UNPACK_DEFINE_H__ -#define MSGPACK_UNPACK_DEFINE_H__ - -#include "sysdep.h" -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - - -#ifndef MSGPACK_EMBED_STACK_SIZE -#define MSGPACK_EMBED_STACK_SIZE 32 -#endif - - -// CS is first byte & 0x1f -typedef enum { - CS_HEADER = 0x00, // nil - - //CS_ = 0x01, - //CS_ = 0x02, // false - //CS_ = 0x03, // true - - CS_BIN_8 = 0x04, - CS_BIN_16 = 0x05, - CS_BIN_32 = 0x06, - - CS_EXT_8 = 0x07, - CS_EXT_16 = 0x08, - CS_EXT_32 = 0x09, - - CS_FLOAT = 0x0a, - CS_DOUBLE = 0x0b, - CS_UINT_8 = 0x0c, - CS_UINT_16 = 0x0d, - CS_UINT_32 = 0x0e, - CS_UINT_64 = 0x0f, - CS_INT_8 = 0x10, - CS_INT_16 = 0x11, - CS_INT_32 = 0x12, - CS_INT_64 = 0x13, - - //CS_FIXEXT1 = 0x14, - //CS_FIXEXT2 = 0x15, - //CS_FIXEXT4 = 0x16, - //CS_FIXEXT8 = 0x17, - //CS_FIXEXT16 = 0x18, - - CS_RAW_8 = 0x19, - CS_RAW_16 = 0x1a, - CS_RAW_32 = 0x1b, - CS_ARRAY_16 = 0x1c, - CS_ARRAY_32 = 0x1d, - CS_MAP_16 = 0x1e, - CS_MAP_32 = 0x1f, - - ACS_RAW_VALUE, - ACS_BIN_VALUE, - ACS_EXT_VALUE, -} msgpack_unpack_state; - - -typedef enum { - CT_ARRAY_ITEM, - CT_MAP_KEY, - CT_MAP_VALUE, -} msgpack_container_type; - - -#ifdef __cplusplus -} -#endif - -#endif /* msgpack/unpack_define.h */ diff --git a/src/borg/cache_sync/unpack_template.h b/src/borg/cache_sync/unpack_template.h deleted file mode 100644 index 39f9f3314..000000000 --- a/src/borg/cache_sync/unpack_template.h +++ /dev/null @@ -1,365 +0,0 @@ -/* - * MessagePack unpacking routine template - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * Copyright (c) 2017 Marian Beermann - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * - * This has been slightly adapted from the vanilla msgpack-{c, python} version. - * Since cache_sync does not intend to build an output data structure, - * msgpack_unpack_object and all of its uses was removed. - */ - -#ifndef USE_CASE_RANGE -#if !defined(_MSC_VER) -#define USE_CASE_RANGE -#endif -#endif - -typedef struct unpack_stack { - size_t size; - size_t count; - unsigned int ct; -} unpack_stack; - -struct unpack_context { - unpack_user user; - unsigned int cs; - unsigned int trail; - unsigned int top; - unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; -}; - -static inline void unpack_init(unpack_context* ctx) -{ - ctx->cs = CS_HEADER; - ctx->trail = 0; - ctx->top = 0; - unpack_init_user_state(&ctx->user); -} - -#define construct 1 - -static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off) -{ - const unsigned char* p = (unsigned char*)data + *off; - const unsigned char* const pe = (unsigned char*)data + len; - const void* n = NULL; - - unsigned int trail = ctx->trail; - unsigned int cs = ctx->cs; - unsigned int top = ctx->top; - unpack_stack* stack = ctx->stack; - unpack_user* user = &ctx->user; - - unpack_stack* c = NULL; - - int ret; - - assert(len >= *off); - -#define construct_cb(name) \ - construct && unpack_callback ## name - -#define push_simple_value(func) \ - if(construct_cb(func)(user) < 0) { goto _failed; } \ - goto _push -#define push_fixed_value(func, arg) \ - if(construct_cb(func)(user, arg) < 0) { goto _failed; } \ - goto _push -#define push_variable_value(func, base, pos, len) \ - if(construct_cb(func)(user, \ - (const char*)base, (const char*)pos, len) < 0) { goto _failed; } \ - goto _push - -#define again_fixed_trail(_cs, trail_len) \ - trail = trail_len; \ - cs = _cs; \ - goto _fixed_trail_again -#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ - trail = trail_len; \ - if(trail == 0) { goto ifzero; } \ - cs = _cs; \ - goto _fixed_trail_again - -#define start_container(func, count_, ct_) \ - if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ - if(construct_cb(func)(user, count_) < 0) { goto _failed; } \ - if((count_) == 0) { \ - if (construct_cb(func##_end)(user) < 0) { goto _failed; } \ - goto _push; } \ - stack[top].ct = ct_; \ - stack[top].size = count_; \ - stack[top].count = 0; \ - ++top; \ - goto _header_again - -#define NEXT_CS(p) ((unsigned int)*p & 0x1f) - -#ifdef USE_CASE_RANGE -#define SWITCH_RANGE_BEGIN switch(*p) { -#define SWITCH_RANGE(FROM, TO) case FROM ... TO: -#define SWITCH_RANGE_DEFAULT default: -#define SWITCH_RANGE_END } -#else -#define SWITCH_RANGE_BEGIN { if(0) { -#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { -#define SWITCH_RANGE_DEFAULT } else { -#define SWITCH_RANGE_END } } -#endif - - if(p == pe) { goto _out; } - do { - switch(cs) { - case CS_HEADER: - SWITCH_RANGE_BEGIN - SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum - push_fixed_value(_uint8, *(uint8_t*)p); - SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum - push_fixed_value(_int8, *(int8_t*)p); - SWITCH_RANGE(0xc0, 0xdf) // Variable - switch(*p) { - case 0xc0: // nil - push_simple_value(_nil); - //case 0xc1: // never used - case 0xc2: // false - push_simple_value(_false); - case 0xc3: // true - push_simple_value(_true); - case 0xc4: // bin 8 - again_fixed_trail(NEXT_CS(p), 1); - case 0xc5: // bin 16 - again_fixed_trail(NEXT_CS(p), 2); - case 0xc6: // bin 32 - again_fixed_trail(NEXT_CS(p), 4); - case 0xc7: // ext 8 - again_fixed_trail(NEXT_CS(p), 1); - case 0xc8: // ext 16 - again_fixed_trail(NEXT_CS(p), 2); - case 0xc9: // ext 32 - again_fixed_trail(NEXT_CS(p), 4); - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - case 0xd4: // fixext 1 - case 0xd5: // fixext 2 - case 0xd6: // fixext 4 - case 0xd7: // fixext 8 - again_fixed_trail_if_zero(ACS_EXT_VALUE, - (1 << (((unsigned int)*p) & 0x03))+1, - _ext_zero); - case 0xd8: // fixext 16 - again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero); - case 0xd9: // str 8 - again_fixed_trail(NEXT_CS(p), 1); - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); - default: - goto _failed; - } - SWITCH_RANGE(0xa0, 0xbf) // FixRaw - again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - SWITCH_RANGE(0x90, 0x9f) // FixArray - start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - SWITCH_RANGE(0x80, 0x8f) // FixMap - start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - - SWITCH_RANGE_DEFAULT - goto _failed; - SWITCH_RANGE_END - // end CS_HEADER - - - _fixed_trail_again: - ++p; // fallthrough - - default: - if((size_t)(pe - p) < trail) { goto _out; } - n = p; p += trail - 1; - switch(cs) { - case CS_EXT_8: - again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero); - case CS_EXT_16: - again_fixed_trail_if_zero(ACS_EXT_VALUE, - _msgpack_load16(uint16_t,n)+1, - _ext_zero); - case CS_EXT_32: - again_fixed_trail_if_zero(ACS_EXT_VALUE, - _msgpack_load32(uint32_t,n)+1, - _ext_zero); - case CS_FLOAT: { - union { uint32_t i; float f; } mem; - mem.i = _msgpack_load32(uint32_t,n); - push_fixed_value(_float, mem.f); } - case CS_DOUBLE: { - union { uint64_t i; double f; } mem; - mem.i = _msgpack_load64(uint64_t,n); -#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi - // https://github.com/msgpack/msgpack-perl/pull/1 - mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); -#endif - push_fixed_value(_double, mem.f); } - case CS_UINT_8: - push_fixed_value(_uint8, *(uint8_t*)n); - case CS_UINT_16: - push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); - case CS_UINT_32: - push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); - case CS_UINT_64: - push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); - - case CS_INT_8: - push_fixed_value(_int8, *(int8_t*)n); - case CS_INT_16: - push_fixed_value(_int16, _msgpack_load16(int16_t,n)); - case CS_INT_32: - push_fixed_value(_int32, _msgpack_load32(int32_t,n)); - case CS_INT_64: - push_fixed_value(_int64, _msgpack_load64(int64_t,n)); - - case CS_BIN_8: - again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero); - case CS_BIN_16: - again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero); - case CS_BIN_32: - again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero); - case ACS_BIN_VALUE: - _bin_zero: - push_variable_value(_bin, data, n, trail); - - case CS_RAW_8: - again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero); - case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); - case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); - case ACS_RAW_VALUE: - _raw_zero: - push_variable_value(_raw, data, n, trail); - - case ACS_EXT_VALUE: - _ext_zero: - push_variable_value(_ext, data, n, trail); - - case CS_ARRAY_16: - start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); - case CS_ARRAY_32: - /* FIXME security guard */ - start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); - - case CS_MAP_16: - start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); - case CS_MAP_32: - /* FIXME security guard */ - start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); - - default: - goto _failed; - } - } - -_push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - if(construct_cb(_array_item)(user, c->count) < 0) { goto _failed; } - if(++c->count == c->size) { - if (construct_cb(_array_end)(user) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, c->count) < 0) { goto _failed; } - if(++c->count == c->size) { - if (construct_cb(_map_end)(user) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; - - default: - goto _failed; - } - -_header_again: - cs = CS_HEADER; - ++p; - } while(p != pe); - goto _out; - - -_finish: - if (!construct) - unpack_callback_nil(user); - ++p; - ret = 1; - /* printf("-- finish --\n"); */ - goto _end; - -_failed: - /* printf("** FAILED **\n"); */ - ret = -1; - goto _end; - -_out: - ret = 0; - goto _end; - -_end: - ctx->cs = cs; - ctx->trail = trail; - ctx->top = top; - *off = p - (const unsigned char*)data; - - return ret; -#undef construct_cb -} - -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END -#undef push_simple_value -#undef push_fixed_value -#undef push_variable_value -#undef again_fixed_trail -#undef again_fixed_trail_if_zero -#undef start_container -#undef construct - -#undef NEXT_CS - -/* vim: set ts=4 sw=4 sts=4 expandtab */ diff --git a/src/borg/hashindex.pyx b/src/borg/hashindex.pyx index 207227035..94149105c 100644 --- a/src/borg/hashindex.pyx +++ b/src/borg/hashindex.pyx @@ -34,19 +34,7 @@ cdef extern from "_hashindex.c": double HASH_MAX_LOAD -cdef extern from "cache_sync/cache_sync.c": - ctypedef struct CacheSyncCtx: - pass - - CacheSyncCtx *cache_sync_init(HashIndex *chunks) - const char *cache_sync_error(const CacheSyncCtx *ctx) - uint64_t cache_sync_num_files_totals(const CacheSyncCtx *ctx) - uint64_t cache_sync_size_totals(const CacheSyncCtx *ctx) - int cache_sync_feed(CacheSyncCtx *ctx, void *data, uint32_t length) - void cache_sync_free(CacheSyncCtx *ctx) - - uint32_t _MAX_VALUE - +_MAX_VALUE = 4294966271UL # 2**32 - 1025 cdef _NoDefault = object() @@ -592,42 +580,3 @@ cdef class ChunkKeyIterator: cdef uint32_t refcount = _le32toh(value[0]) assert refcount <= _MAX_VALUE, "invalid reference count" return (self.key)[:self.key_size], ChunkIndexEntry(refcount, _le32toh(value[1])) - - -cdef Py_buffer ro_buffer(object data) except *: - cdef Py_buffer view - PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) - return view - - -cdef class CacheSynchronizer: - cdef ChunkIndex chunks - cdef CacheSyncCtx *sync - - def __cinit__(self, chunks): - self.chunks = chunks - self.sync = cache_sync_init(self.chunks.index) - if not self.sync: - raise Exception('cache_sync_init failed') - - def __dealloc__(self): - if self.sync: - cache_sync_free(self.sync) - - def feed(self, chunk): - cdef Py_buffer chunk_buf = ro_buffer(chunk) - cdef int rc - rc = cache_sync_feed(self.sync, chunk_buf.buf, chunk_buf.len) - PyBuffer_Release(&chunk_buf) - if not rc: - error = cache_sync_error(self.sync) - if error != NULL: - raise ValueError('cache_sync_feed failed: ' + error.decode('ascii')) - - @property - def num_files_totals(self): - return cache_sync_num_files_totals(self.sync) - - @property - def size_totals(self): - return cache_sync_size_totals(self.sync) diff --git a/src/borg/testsuite/cache.py b/src/borg/testsuite/cache.py index f9de6ccc7..28846d0ad 100644 --- a/src/borg/testsuite/cache.py +++ b/src/borg/testsuite/cache.py @@ -10,156 +10,11 @@ from .key import TestKey from ..archive import Statistics from ..cache import AdHocCache from ..crypto.key import AESOCBRepoKey -from ..hashindex import ChunkIndex, CacheSynchronizer +from ..hashindex import ChunkIndex from ..manifest import Manifest from ..repository3 import Repository3 -class TestCacheSynchronizer: - @pytest.fixture - def index(self): - return ChunkIndex() - - @pytest.fixture - def sync(self, index): - return CacheSynchronizer(index) - - def test_no_chunks(self, index, sync): - data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": []}) - sync.feed(data) - assert not len(index) - - def test_simple(self, index, sync): - data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]}) - sync.feed(data) - assert len(index) == 2 - assert index[H(1)] == (1, 1) - assert index[H(2)] == (1, 2) - - def test_multiple(self, index, sync): - data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]}) - data += packb({"xattrs": {"security.foo": "bar", "chunks": "123456"}, "stuff": [(1, 2, 3)]}) - data += packb( - { - "xattrs": {"security.foo": "bar", "chunks": "123456"}, - "chunks": [(H(1), 1), (H(2), 2)], - "stuff": [(1, 2, 3)], - } - ) - data += packb({"chunks": [(H(3), 1)]}) - data += packb({"chunks": [(H(1), 1)]}) - - part1 = data[:70] - part2 = data[70:120] - part3 = data[120:] - sync.feed(part1) - sync.feed(part2) - sync.feed(part3) - assert len(index) == 3 - assert index[H(1)] == (3, 1) - assert index[H(2)] == (2, 2) - assert index[H(3)] == (1, 1) - - @pytest.mark.parametrize( - "elem,error", - ( - ({1: 2}, "Unexpected object: map"), - ( - bytes(213), - ["Unexpected bytes in chunks structure", "Incorrect key length"], # structure 2/3 - ), # structure 3/3 - (1, "Unexpected object: integer"), - (1.0, "Unexpected object: double"), - (True, "Unexpected object: true"), - (False, "Unexpected object: false"), - (None, "Unexpected object: nil"), - ), - ids=["map", "bytes", "int", "double", "true", "false", "none"], - ) - @pytest.mark.parametrize( - "structure", - (lambda elem: {"chunks": elem}, lambda elem: {"chunks": [elem]}, lambda elem: {"chunks": [(elem, 1)]}), - ) - def test_corrupted(self, sync, structure, elem, error): - packed = packb(structure(elem)) - with pytest.raises(ValueError) as excinfo: - sync.feed(packed) - if isinstance(error, str): - error = [error] - possible_errors = ["cache_sync_feed failed: " + error for error in error] - assert str(excinfo.value) in possible_errors - - @pytest.mark.parametrize( - "data,error", - ( - # Incorrect tuple length - ({"chunks": [(bytes(32), 2, 3, 4)]}, "Invalid chunk list entry length"), - ({"chunks": [(bytes(32),)]}, "Invalid chunk list entry length"), - # Incorrect types - ({"chunks": [(1, 2)]}, "Unexpected object: integer"), - ({"chunks": [(1, bytes(32))]}, "Unexpected object: integer"), - ({"chunks": [(bytes(32), 1.0)]}, "Unexpected object: double"), - ), - ) - def test_corrupted_ancillary(self, index, sync, data, error): - packed = packb(data) - with pytest.raises(ValueError) as excinfo: - sync.feed(packed) - assert str(excinfo.value) == "cache_sync_feed failed: " + error - - def make_index_with_refcount(self, refcount): - index_data = io.BytesIO() - index_data.write(b"BORG2IDX") - # version - index_data.write((2).to_bytes(4, "little")) - # num_entries - index_data.write((1).to_bytes(4, "little")) - # num_buckets - index_data.write((1).to_bytes(4, "little")) - # num_empty - index_data.write((0).to_bytes(4, "little")) - # key_size - index_data.write((32).to_bytes(4, "little")) - # value_size - index_data.write((3 * 4).to_bytes(4, "little")) - # reserved - index_data.write(bytes(1024 - 32)) - - index_data.write(H(0)) - index_data.write(refcount.to_bytes(4, "little")) - index_data.write((1234).to_bytes(4, "little")) - index_data.write((5678).to_bytes(4, "little")) - - index_data.seek(0) - index = ChunkIndex.read(index_data) - return index - - def test_corrupted_refcount(self): - index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE + 1) - sync = CacheSynchronizer(index) - data = packb({"chunks": [(H(0), 1)]}) - with pytest.raises(ValueError) as excinfo: - sync.feed(data) - assert str(excinfo.value) == "cache_sync_feed failed: invalid reference count" - - def test_refcount_max_value(self): - index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE) - sync = CacheSynchronizer(index) - data = packb({"chunks": [(H(0), 1)]}) - sync.feed(data) - assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234) - - def test_refcount_one_below_max_value(self): - index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE - 1) - sync = CacheSynchronizer(index) - data = packb({"chunks": [(H(0), 1)]}) - sync.feed(data) - # Incremented to maximum - assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234) - sync.feed(data) - assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234) - - class TestAdHocCache: @pytest.fixture def repository(self, tmpdir):