ITS#9596 First take on Python test suite

tests/python/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+*.pyc

tests/python/backends.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# This work is part of OpenLDAP Software <http://www.openldap.org/>.
+#
+# Copyright 2021 The OpenLDAP Foundation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted only as authorized by the OpenLDAP
+# Public License.
+#
+# A copy of this license is available in the file LICENSE in the
+# top-level directory of the distribution or, alternatively, at
+# <http://www.OpenLDAP.org/license.html>.
+#
+# ACKNOWLEDGEMENTS:
+# This work was initially developed by Ondřej Kuzník
+# for inclusion in OpenLDAP Software.
+"""
+OpenLDAP fixtures for backends
+"""
+
+import ldap0
+import logging
+import os
+import pathlib
+import pytest
+import secrets
+import tempfile
+
+from ldap0.controls.readentry import PostReadControl
+
+from .slapd import server
+
+
+SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute()
+BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute()
+
+
+logger = logging.getLogger(__name__)
+
+
+class Database:
+    have_directory = True
+
+    def __init__(self, server, suffix, backend):
+        self.server = server
+        self.suffix = suffix
+        self.rootdn = suffix
+        self.secret = secrets.token_urlsafe()
+        self.overlays = []
+
+        if suffix in server.suffixes:
+            raise RuntimeError(f"Suffix {suffix} already configured in server")
+
+        if self.have_directory:
+            self.directory = tempfile.TemporaryDirectory(dir=server.home)
+
+        conn = server.connect()
+        conn.simple_bind_s("cn=config", server.secret)
+
+        # We're just after the generated DN, no other attributes at the moment
+        control = PostReadControl(True, [])
+
+        result = conn.add_s(
+            f"olcDatabase={backend},cn=config", self._entry(),
+            req_ctrls=[control])
+        dn = result.ctrls[0].res.dn_s
+
+        self.dn = dn
+        server.suffixes[suffix] = self
+
+    def _entry(self):
+        entry = {
+            "objectclass": [self.objectclass.encode()],
+            "olcSuffix": [self.suffix.encode()],
+            "olcRootDN": [self.suffix.encode()],
+            "olcRootPW": [self.secret.encode()],
+        }
+        if self.have_directory:
+            entry["olcDbDirectory"] = [self.directory.name.encode()]
+        return entry
+
+
+class MDB(Database):
+    have_directory = True
+    objectclass = "olcMdbConfig"
+
+    _size = 10 * (1024 ** 3)
+
+    def __init__(self, server, suffix):
+        super().__init__(server, suffix, "mdb")
+
+    def _entry(self):
+        entry = {
+            "olcDbMaxSize": [str(self._size).encode()],
+        }
+        return {**super()._entry(), **entry}
+
+
+class LDAP(Database):
+    have_directory = False
+    objectclass = "olcLDAPConfig"
+
+    def __init__(self, server, suffix, uris):
+        self.uris = uris
+        super().__init__(server, suffix, "ldap")
+
+    def _entry(self):
+        entry = {
+            "olcDbURI": [" ".join(self.uris).encode()],
+        }
+        return {**super()._entry(), **entry}
+
+
+backend_types = {
+    "mdb": MDB,
+    "ldap": LDAP,
+}
+
+
+@pytest.fixture(scope="class")
+def db(request, server):
+    marker = request.node.get_closest_marker("db")
+    database_type = marker.args[0] if marker else "mdb"
+    klass = backend_types[database_type]
+
+    conn = server.connect()
+    conn.simple_bind_s("cn=config", server.secret)
+
+    db = klass(server, "cn=test")
+    yield db
+
+    conn.delete_s(db.dn)
+
+
+class TestDB:
+    def test_db_setup(self, db):
+        pass

tests/python/conftest.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# This work is part of OpenLDAP Software <http://www.openldap.org/>.
+#
+# Copyright 2021 The OpenLDAP Foundation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted only as authorized by the OpenLDAP
+# Public License.
+#
+# A copy of this license is available in the file LICENSE in the
+# top-level directory of the distribution or, alternatively, at
+# <http://www.OpenLDAP.org/license.html>.
+#
+# ACKNOWLEDGEMENTS:
+# This work was initially developed by Ondřej Kuzník
+# for inclusion in OpenLDAP Software.
+"""
+OpenLDAP test suite fixtures
+"""
+
+import pytest
+
+from .slapd import temp, server_factory, server
+from .backends import db

tests/python/overlays.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# This work is part of OpenLDAP Software <http://www.openldap.org/>.
+#
+# Copyright 2021 The OpenLDAP Foundation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted only as authorized by the OpenLDAP
+# Public License.
+#
+# A copy of this license is available in the file LICENSE in the
+# top-level directory of the distribution or, alternatively, at
+# <http://www.OpenLDAP.org/license.html>.
+#
+# ACKNOWLEDGEMENTS:
+# This work was initially developed by Ondřej Kuzník
+# for inclusion in OpenLDAP Software.
+"""
+OpenLDAP fixtures for overlays
+"""
+
+import logging
+import os
+import pathlib
+
+from ldap0.controls.readentry import PostReadControl
+
+
+SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute()
+BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute()
+
+
+logger = logging.getLogger(__name__)
+
+
+class Overlay:
+    def __init__(self, database, overlay, order=-1):
+        self.database = database
+        server = database.server
+
+        conn = server.connect()
+        conn.simple_bind_s("cn=config", server.secret)
+
+        if isinstance(overlay, pathlib.Path):
+            overlay_name = overlay.stem
+        else:
+            overlay_name = overlay
+            overlay = BUILDROOT/"servers"/"slapd"/"overlays"/overlay_name
+
+        server.load_module(overlay)
+
+        # We're just after the generated DN, no other attributes at the moment
+        control = PostReadControl(True, [])
+
+        result = conn.add_s(
+            f"olcOverlay={overlay_name},{database.dn}", self._entry(),
+            req_ctrls=[control])
+        self.dn = result.ctrls[0].res.dn_s
+
+        if order == -1:
+            database.overlays.append(self)
+        else:
+            raise NotImplementedError
+            database.overlays.insert(order, self)
+
+    def _entry(self):
+        entry = {
+            "objectclass": [self.objectclass.encode()],
+        }
+        return entry

tests/python/slapd.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# This work is part of OpenLDAP Software <http://www.openldap.org/>.
+#
+# Copyright 2021 The OpenLDAP Foundation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted only as authorized by the OpenLDAP
+# Public License.
+#
+# A copy of this license is available in the file LICENSE in the
+# top-level directory of the distribution or, alternatively, at
+# <http://www.OpenLDAP.org/license.html>.
+#
+# ACKNOWLEDGEMENTS:
+# This work was initially developed by Ondřej Kuzník
+# for inclusion in OpenLDAP Software.
+"""
+OpenLDAP server fixtures
+"""
+
+import ldap0
+import ldapurl
+import logging
+import os
+import pathlib
+import pytest
+import re
+import secrets
+import signal
+import socket
+import subprocess
+import tempfile
+import textwrap
+
+from ldap0.ldapobject import LDAPObject
+
+
+SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute()
+BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute()
+
+
+logger = logging.getLogger(__name__)
+
+
+class Server:
+    def __init__(self, where, manager, cnconfig=True, schemas=None):
+        self.path = where
+        self.home = pathlib.Path(self.path.name)
+        self.executable = BUILDROOT/'servers'/'slapd'/'slapd'
+
+        self.manager = manager
+        self.cnconfig = cnconfig
+
+        self.token = secrets.token_urlsafe()
+        self.secret = None
+        self.level = "-1"
+        self.port = 0
+        self.pid = None
+
+        if schemas is None:
+            schemas = ["core", "cosine", "inetorgperson", "openldap", "nis"]
+
+        if cnconfig and not (self.home/'slapd.d').is_dir():
+            self.create_config(schemas)
+        elif not cnconfig and not (self.home/'slapd.conf').is_file():
+            self.create_config(schemas)
+
+        self.process = None
+        self.schema = []
+        self.suffixes = {}
+
+    def create_config(self, schemas):
+        mod_harness = BUILDROOT/"tests"/"modules"/"mod-harness"/"mod_harness"
+        schemadir = SOURCEROOT/"servers"/"slapd"/"schema"
+        if not self.secret:
+            self.secret = secrets.token_urlsafe()
+
+        if self.cnconfig:
+            confdir = self.home/'slapd.d'
+            confdir.mkdir()
+            includes = []
+
+            config = """
+                dn: cn=config
+                objectClass: olcGlobal
+                cn: config
+
+                dn: cn=module{{0}},cn=config
+                objectClass: olcModuleList
+                olcModuleLoad: {mod_harness}
+
+                dn: cn=schema,cn=config
+                objectClass: olcSchemaConfig
+                cn: schema
+
+                dn: olcBackend={{0}}harness,cn=config
+                objectClass: olcBkHarnessConfig
+                olcBkHarnessHost: {self.manager.host}
+                olcBkHarnessPort: {self.manager.port}
+                olcBkHarnessIdentifier: {self.token}
+
+                dn: olcDatabase={{0}}config,cn=config
+                objectClass: olcDatabaseConfig
+                olcRootPW: {self.secret}
+            """.format(self=self, mod_harness=mod_harness)
+
+            for schema in schemas:
+                if not isinstance(schema, pathlib.Path):
+                    schema = schemadir / (schema + ".ldif")
+                includes.append(f"include: file://{schema}")
+
+            config = "\n".join([textwrap.dedent(config), "\n", *includes])
+
+            args = [self.executable, '-T', 'add', '-d', self.level,
+                    '-n0', '-F', confdir]
+            args = [str(arg) for arg in args]
+            subprocess.run(args, capture_output=True, check=True,
+                           cwd=self.home, text=True, input=config)
+        else:
+            with open(self.home/'slapd.conf', mode='w') as config:
+                config.write(textwrap.dedent("""
+                    moduleload {mod_harness}
+
+                    backend harness
+                    host {self.manager.host}
+                    port {self.manager.port}
+                    identifier {self.token}
+
+                    database config
+                    rootpw {self.secret}
+                """.format(self=self, mod_harness=mod_harness)))
+
+                includes = []
+                for schema in schemas:
+                    if not isinstance(schema, pathlib.Path):
+                        schema = schemadir / (schema + ".schema")
+                    includes.append(f"include {schema}\n")
+
+                config.write("".join(includes))
+
+    def test(self):
+        args = [self.executable, '-T', 'test', '-d', self.level]
+        if self.cnconfig:
+            args += ['-F', self.home/'slapd.d']
+        else:
+            args += ['-f', self.home/'slapd.conf']
+
+        args = [str(arg) for arg in args]
+        return subprocess.run(args, capture_output=True, check=True,
+                              cwd=self.home)
+
+    def start(self, port=None):
+        if self.process:
+            raise RuntimeError("process %d still running" % self.process.pid)
+
+        self.test()
+
+        if port is not None:
+            self.port = port
+
+        listeners = [
+            'ldapi://socket',
+            'ldap://localhost:%d' % self.port,
+        ]
+        args = [self.executable, '-d', self.level]
+        if self.cnconfig:
+            args += ['-F', self.home/'slapd.d']
+        else:
+            args += ['-f', self.home/'slapd.conf']
+        args += ['-h', ' '.join(listeners)]
+
+        with open(self.home/'slapd.log', 'a+') as log:
+            args = [str(arg) for arg in args]
+            self.process = subprocess.Popen(args, stderr=log, cwd=self.home)
+        self.log = open(self.home/'slapd.log', 'r+')
+
+        self.connection, self.pid = self.manager.wait(self.token)
+
+        line = self.connection.readline().strip()
+        while line:
+            if line == 'SLAPD READY':
+                break
+            elif line.startswith("URI="):
+                uri, name = line[4:].split()
+            line = self.connection.readline().strip()
+
+    def stop(self):
+        if self.process:
+            os.kill(self.pid, signal.SIGHUP)
+            self.process.terminate()
+            self.process.wait()
+        self.process = None
+
+    def connect(self):
+        return LDAPObject(str(self.uri))
+
+    def load_module(self, module):
+        if not self.cnconfig:
+            raise NotImplementedError
+
+        if not isinstance(module, pathlib.Path):
+            raise NotImplementedError
+        module_name = module.stem
+
+        conn = self.connect()
+        conn.simple_bind_s('cn=config', self.secret)
+
+        moduleload_object = None
+        for entry in conn.search_s('cn=config', ldap0.SCOPE_SUBTREE,
+                                   'objectclass=olcModuleList',
+                                   ['olcModuleLoad']):
+            if not moduleload_object:
+                moduleload_object = entry.dn_s
+            for value in entry.entry_s.get('olcModuleLoad', []):
+                if value[0] == '{':
+                    value = value[value.find('}')+1:]
+                if pathlib.Path(value).stem == module_name:
+                    logger.warning("Module %s already loaded, ignoring",
+                                   module_name)
+                    return
+
+        if moduleload_object:
+            conn.modify_s(
+                moduleload_object,
+                [(ldap0.MOD_ADD, b'olcModuleLoad', [str(module).encode()])])
+        else:
+            conn.add_s('cn=module,cn=config',
+                       {'objectClass': [b'olcModuleList'],
+                        'olcModuleLoad': [str(module).encode()]})
+
+    @property
+    def uri(self):
+        return ldapurl.LDAPUrl(urlscheme="ldapi",
+                               hostport=str(self.home/'socket'))
+
+
+class ServerManager:
+    def __init__(self, tmp_path):
+        self.tmpdir = tmp_path
+        self.waiter = socket.create_server(('localhost', 0))
+        self.address = self.waiter.getsockname()
+
+    @property
+    def host(self):
+        return self.address[0]
+
+    @property
+    def port(self):
+        return self.address[1]
+
+    def new_server(self):
+        path = tempfile.TemporaryDirectory(dir=self.tmpdir)
+        return Server(path, self)
+
+    def wait(self, token):
+        s, _ = self.waiter.accept()
+        f = s.makefile('r')
+        response = f.readline().split()
+        if response[0] != 'PID':
+            response.close()
+            raise RuntimeError("Unexpected response")
+        if response[2] != token:
+            raise NotImplementedError("Concurrent startup not implemented yet")
+        return f, int(response[1])
+
+
+@pytest.fixture(scope="module")
+def temp(request, tmp_path_factory):
+    # Stolen from pytest.tmpdir._mk_tmp
+    name = request.node.name
+    name = re.sub(r"[\W]", "_", name)
+    MAXVAL = 30
+    name = name[:MAXVAL]
+    return tmp_path_factory.mktemp(name, numbered=True)
+
+
+@pytest.fixture(scope="module")
+def server_factory(temp):
+    return ServerManager(temp)
+
+
+@pytest.fixture(scope="class")
+def server(server_factory):
+    server = server_factory.new_server()
+    server.start()
+    yield server
+    server.stop()
+    server.path.cleanup()
+
+
+def test_rootdse(server):
+    conn = server.connect()
+    conn.search_s("", scope=ldap0.SCOPE_BASE)

tests/python/syncrepl.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# This work is part of OpenLDAP Software <http://www.openldap.org/>.
+#
+# Copyright 2021 The OpenLDAP Foundation.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted only as authorized by the OpenLDAP
+# Public License.
+#
+# A copy of this license is available in the file LICENSE in the
+# top-level directory of the distribution or, alternatively, at
+# <http://www.OpenLDAP.org/license.html>.
+#
+# ACKNOWLEDGEMENTS:
+# This work was initially developed by Ondřej Kuzník
+# for inclusion in OpenLDAP Software.
+"""
+OpenLDAP fixtures for overlays
+"""
+
+import ldap0
+import logging
+import os
+import pathlib
+import pytest
+import subprocess
+
+
+from .slapd import server
+from .backends import db, backend_types
+from .overlays import Overlay
+
+
+SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute()
+BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute()
+
+
+logger = logging.getLogger(__name__)
+
+
+class Syncprov(Overlay):
+    objectclass = 'olcSyncprovConfig'
+
+    def __init__(self, backend, *args, **kwargs):
+        super().__init__(backend, 'syncprov', *args, **kwargs)
+
+
+@pytest.fixture(scope="class")
+def provider(request, db):
+    conn = server.connect()
+    conn.simple_bind_s("cn=config", server.secret)
+
+    syncprov = Syncprov(db)
+    yield db.server
+
+    conn.delete_s(syncprov.dn)
+
+
+@pytest.fixture(scope="class")
+def replica(request, server_factory, provider):
+    raise NotImplementedError
+
+
+@pytest.fixture(scope="class")
+def mmr(request, server_factory):
+    mmr_marker = request.node.get_closest_marker("mmr")
+    mmr_args = mmr_marker and mmr_marker.args or {}
+    server_count = mmr_args.get("mmr", 4)
+    serverids = mmr_args.get("serverids", range(1, server_count+1))
+    server_connections = mmr_args.get("connections") or \
+        {consumer: {provider for provider in serverids if provider != consumer}
+            for consumer in serverids}
+
+    database_marker = request.node.get_closest_marker("db")
+    database_type = database_marker.args[0] if database_marker else "mdb"
+    db_class = backend_types[database_type]
+
+    servers = {}
+    connections = {}
+    for serverid in serverids:
+        server = server_factory.new_server()
+        server.start()
+        conn = server.connect()
+        conn.simple_bind_s("cn=config", server.secret)
+
+        conn.modify_s("cn=config", [
+                (ldap0.MOD_REPLACE, b"olcServerId", [str(serverid).encode()])])
+
+        server.serverid = serverid
+        servers[serverid] = server
+        connections[serverid] = conn
+
+        db = db_class(server, "dc=example,dc=com")
+        syncprov = Syncprov(db)
+
+    for serverid, server in servers.items():
+        suffix = db.suffix
+
+        syncrepl = []
+        for providerid in server_connections[serverid]:
+            provider = servers[providerid]
+            db = provider.suffixes[suffix]
+            syncrepl.append((
+                f'rid={providerid} provider={provider.uri} '
+                f'searchbase="{db.suffix}" '
+                f'type=refreshAndPersist retry="1 +" '
+                f'bindmethod=simple '
+                f'binddn="{db.suffix}" credentials="{db.secret}"').encode())
+
+        connections[serverid].modify_s(db.dn, [
+            (ldap0.MOD_REPLACE, b"olcSyncrepl", syncrepl),
+            (ldap0.MOD_REPLACE, b"olcMultiprovider", [b"TRUE"])])
+
+    yield servers
+
+    for serverid, server in servers.items():
+        server.stop()
+        server.path.cleanup()
+
+
+# TODO: after we switch to asyncio, make use of the syncmonitor module
+# directly.
+# We should even wrap this in a class to allow finer grained control
+# over the behaviour like waiting for partial syncs etc.
+def wait_for_resync(searchbase, servers, timeout=30):
+    subprocess.check_call(["synccheck", "-p", "--base", searchbase,
+                           "--timeout", str(timeout),
+                           *[str(server.uri) for server in servers],
+                           ], timeout=timeout+5)
+
+
+def test_mmr(mmr):
+    suffix = "dc=example,dc=com"
+    entries_added = set()
+
+    connections = []
+    for serverid, server in mmr.items():
+        db = server.suffixes[suffix]
+        conn = server.connect()
+        conn.simple_bind_s(db.rootdn, db.secret)
+
+        if not entries_added:
+            conn.add_s(suffix, {
+                "objectClass": [b"organization",
+                                b"domainRelatedObject",
+                                b"dcobject"],
+                "o": [b"Example, Inc."],
+                "associatedDomain": [b"example.com"]})
+            entries_added.add(suffix)
+            # Make sure all hosts have the suffix entry
+            wait_for_resync(suffix, mmr.values())
+
+        dn = f"cn=entry{serverid},{suffix}"
+        conn.add_s(dn, {"objectClass": [b"device"],
+                        "description": [(f"Entry created on serverid "
+                                         f"{serverid}").encode()]})
+        entries_added.add(dn)
+        connections.append(conn)
+
+    wait_for_resync(suffix, mmr.values())
+
+    for conn in connections:
+        result = conn.search_s(suffix, ldap0.SCOPE_SUBTREE, attrlist=['1.1'])
+        dns = {entry.dn_s for entry in result}
+        assert dns == entries_added, \
+                f"Server {serverid} contents do not match expectations"