diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py
index 00a3a2bd9..81906bf27 100644
--- a/borg/testsuite/archiver.py
+++ b/borg/testsuite/archiver.py
@@ -71,10 +71,15 @@ class environment_variable:
                 os.environ[k] = v
 
 
-def exec_cmd(*args, archiver=None, fork=False, **kw):
+def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
     if fork:
         try:
-            borg = (sys.executable, '-m', 'borg.archiver')
+            if exe is None:
+                borg = (sys.executable, '-m', 'borg.archiver')
+            elif isinstance(exe, str):
+                borg = (exe, )
+            elif not isinstance(exe, tuple):
+                raise ValueError('exe must be None, a tuple or a str')
             output = subprocess.check_output(borg + args)
             ret = 0
         except subprocess.CalledProcessError as e:
diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py
index 5b5c38634..d7bf95bb0 100644
--- a/borg/testsuite/benchmark.py
+++ b/borg/testsuite/benchmark.py
@@ -13,10 +13,17 @@ import pytest
 from .archiver import changedir, exec_cmd
 
 
-# TODO: use fixture params to test python code and binary
-@pytest.fixture
-def cmd():
-    return exec_cmd
+@pytest.fixture(params=['python', 'binary'])
+def cmd(request):
+    if request.param == 'python':
+        exe = None
+    elif request.param == 'binary':
+        exe = 'borg.exe'
+    else:
+        raise ValueError("param must be 'python' or 'binary'")
+    def exec_fn(*args, **kw):
+        return exec_cmd(*args, exe=exe, fork=True, **kw)
+    return exec_fn
 
 
 @pytest.yield_fixture