diff --git a/setup.py b/setup.py
index 631b927fa..58675c875 100644
--- a/setup.py
+++ b/setup.py
@@ -811,10 +811,11 @@ if not on_rtd:
     crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
                                                system_prefix=libb2_prefix, system=libb2_system,
                                                **crypto_ext_kwargs)
-
-    crypto_ext_kwargs = setup_xxhash.xxhash_ext_kwargs(bundled_path='src/borg/algorithms/xxh64',
+    checksums_ext_kwargs = dict(sources=[checksums_source], include_dirs=include_dirs, library_dirs=library_dirs,
+                                define_macros=define_macros)
+    checksums_ext_kwargs = setup_xxhash.xxhash_ext_kwargs(bundled_path='src/borg/algorithms/xxh64',
                                                system_prefix=libxxhash_prefix, system=libxxhash_system,
-                                               **crypto_ext_kwargs)
+                                               **checksums_ext_kwargs)
 
     msgpack_macros = []  # setup.py of msgpack 0.5.6 defines __LITTLE_ENDIAN__ / __BIG_ENDIAN__ - which
                          # leads to troubles when trying cross-platform builds, see borg issue #6105.
@@ -844,7 +845,7 @@ if not on_rtd:
         Extension('borg.hashindex', [hashindex_source]),
         Extension('borg.item', [item_source]),
         Extension('borg.chunker', [chunker_source]),
-        Extension('borg.algorithms.checksums', [checksums_source]),
+        Extension('borg.algorithms.checksums', **checksums_ext_kwargs),
     ]
     if not sys.platform.startswith(('win32', )):
         ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))