certbot/certbot-apache/certbot_apache/_internal/prefetch_ocsp.py

"""A mixin class for OCSP response prefetching for Apache plugin.

The OCSP prefetching functionality solves multiple issues in Apache httpd
that make using OCSP must-staple error prone. See
https://blog.hboeck.de/archives/886-The-Problem-with-OCSP-Stapling-and-Must-Staple-and-why-Certificate-Revocation-is-still-broken.html
for more details on what these problems are, however, the solution implemented
here is to have Certbot fetch the OCSP response for Apache to staple. This
happens whenever a certificate is obtained or updated and periodically when
`certbot renew` is run even if the certificate is not renewed.

The prefetching functionality works by storing values using
certbot.plugins.storage noting which certificates Certbot should keep OCSP
staples (OCSP responses) updated for alongside of the information when the last
response was updated by Certbot.

When Certbot is invoked, typically by scheduled "certbot renew" and the
TTL from the "lastupdate" value in PluginStorage entry has expired,
Certbot then proceeds to fetch a new OCSP response from the OCSP servers
pointed by the certificate. The OCSP response is validated and if valid, stored
for Apache to use.

Apache is configured to cache the OCSP responses it has stapled in a DBM file.
This file was not intended to be modified by external programs so much of the
knowledge here is based on testing and reading the Apache source code. This
fact also complicates things like handling race conditions and compatibility
with the different DBM formats that Apache can use as described below.

The DBM cache file used by Apache is a lightweight key-value storage.  For OCSP
response caching, the sha1 hash of certificate fingerprint is used as a key.
The value consists of expiry time as timestamp in microseconds, \x01 delimiter
and the raw OCSP response.

When Certbot modifies to this file, a high internal cache expiry value is set
for Apache in order to make it to not to discard the stored response and try to
renew the staple itself letting Certbot to renew it on its subsequent run
instead. When this file is read or modified by Certbot, extra care is taken to
try and avoid race conditions with Apache the best we can. Apache does not need
to be reloaded/restarted to pick up changes to this file.

When restarting Apache, Certbot backups the current OCSP response
cache, and restores it after the restart has happened. This is
done because Apache deletes and then recreates the file upon
restart.

There are multiple incompatible libraries for DBM files which Apache can be
configured to use. To deal with this, as of writing this, this mixin is only
used in the Debian specific override classes and we expect Apache to use the
DBM libaries it was configured to use in the Debian Apache packages at the time
this feature was written. Python itself may or may not have support for this
DBM format and if it does not, this feature is not offered to the user.

"""

from datetime import datetime
import logging
import time

from acme.magic_typing import Dict, Union  # pylint: disable=unused-import, no-name-in-module

from certbot import errors
from certbot import interfaces
from certbot import ocsp
from certbot.plugins.enhancements import OCSPPrefetchEnhancement

from certbot.compat import filesystem
from certbot.compat import os
from certbot_apache._internal import apache_util
from certbot_apache._internal import constants

logger = logging.getLogger(__name__)


class DBMHandler(object):
    """Context manager to handle DBM file reads and writes"""

    def __init__(self, filename, mode):
        self.filename = filename
        self.filemode = mode
        self.bsddb = False
        self.database = None

    def __enter__(self):
        """Open the DBM file and return the filehandle"""

        try:
            import bsddb
            self.bsddb = True
            try:
                self.database = bsddb.hashopen(self.filename, self.filemode)
            except Exception:
                raise errors.PluginError("Unable to open dbm database file.")
        except ImportError:
            # Python3 doesn't have bsddb module, so we use dbm.ndbm instead
            import dbm
            if self.filename.endswith(".db"):
                self.filename = self.filename[:-3]
            try:
                self.database = dbm.ndbm.open(self.filename, self.filemode)  # pylint: disable=no-member
            except Exception:
                # This is raised if a file cannot be found
                raise errors.PluginError("Unable to open dbm database file.")
        return self.database

    def __exit__(self, *args):
        """Close the DBM file"""
        if self.bsddb:
            self.database.sync()
        self.database.close()


class OCSPPrefetchMixin(object):
    """OCSPPrefetchMixin implements OCSP response prefetching"""

    def __init__(self, *args, **kwargs):
        self._ocsp_prefetch = {}  # type: Dict[str, Dict[str, Union[str, float]]]
        # This is required because of python super() call chain.
        # Additionally, mypy isn't able to figure the chain out and needs to be
        # disabled for this line. See https://github.com/python/mypy/issues/5887
        super(OCSPPrefetchMixin, self).__init__(*args, **kwargs)  # type: ignore
        self._ocsp_store = os.path.join(self.config.work_dir, "ocsp", "ocsp_cache.db")
        self._ocsp_work = os.path.join(self.config.work_dir, "ocsp_work", "ocsp_cache.db")

    def _ensure_ocsp_dirs(self):
        """Makes sure that the OCSP directory paths exist."""
        for path in [os.path.dirname(self._ocsp_work),
                     os.path.dirname(self._ocsp_store)]:
            if not os.path.isdir(path):
                filesystem.makedirs(path, 0o755)

    def _ensure_ocsp_prefetch_compatibility(self):
        """Make sure that the operating system supports the required libraries
        to manage Apache DBM files.

        :raises: errors.NotSupportedError
        """
        if not ocsp.CRYPTOGRAPHY_OCSP_AVAILABLE:
            raise errors.NotSupportedError(
                "You need version 2.5+ of the Python library cryptography to"
                " use this feature.")
        try:
            import bsddb  # pylint: disable=unused-variable
        except ImportError:
            import dbm
            if not hasattr(dbm, 'ndbm') or dbm.ndbm.library != 'Berkeley DB':  # pylint: disable=no-member
                msg = ("Unfortunately your operating system does not have a "
                       "compatible database module available for managing "
                       "Apache OCSP stapling cache database.")
                raise errors.NotSupportedError(msg)

    def _ocsp_refresh_needed(self, lastupdate):
        """Refreshes OCSP response for a certificate if it's due

        :param int lastupdate: Last update timestamp from pluginstorage entry

        :returns: If OCSP response was updated
        :rtype: bool

        """
        ttl = lastupdate + constants.OCSP_INTERNAL_TTL
        if ttl < time.time():
            return True
        return False


    def _ocsp_refresh(self, cert_path, chain_path):
        """Refresh the OCSP response for a certificate

        :param str cert_path: Filesystem path to certificate file
        :param str chain_path: Filesystem path to certificate chain file

        :raises .errors.PluginError: If the OCSP response should not be
            configured for use with Apache

        :raises OCSPCertificateError: If the given certificate should be
            removed from the OCSP prefetch pool.
        """

        self._ensure_ocsp_dirs()
        # Ensure cert_path exists before trying to use it.
        if not os.path.isfile(cert_path):
            raise OCSPCertificateError("Certificate has been removed from the system.")
        handler = ocsp.RevocationChecker()
        ocsp_response = handler.ocsp_response_by_paths(cert_path, chain_path)
        if ocsp_response is None:
            raise errors.PluginError("Unable to obtain an OCSP response.")
        elif ocsp_response.certificate_status == interfaces.OCSPCertStatus.REVOKED:
            raise OCSPCertificateError("Certificate has been revoked.")
        elif ocsp_response.certificate_status == interfaces.OCSPCertStatus.UNKNOWN:
            raise OCSPCertificateError("Certificate is unknown to the OCSP responder.")
        else:
            # Guaranteed good response
            cert_sha = apache_util.certid_sha1(cert_path)
            # dbm.open automatically adds the file extension
            self._write_to_dbm(self._ocsp_store, cert_sha, self._ocsp_response_dbm(ocsp_response))

    def _write_to_dbm(self, filename, key, value):
        """Helper method to write an OCSP response cache value to DBM.

        :param filename: DBM database filename
        :param bytes key: Database key name
        :param bytes value: Database entry value

        :raises .errors.PluginError: If there was an error copying
            Apache's DBM file

        """
        tmp_file = os.path.join(
            os.path.dirname(self._ocsp_work),
            "tmp_" + os.path.basename(filename)
        )

        apache_util.safe_copy(filename, tmp_file)

        with DBMHandler(tmp_file, 'w') as db:
            db[key] = value

        filesystem.replace(tmp_file, filename)

    def _ocsp_ttl(self, next_update):
        """Calculates Apache internal TTL for the next OCSP staple
        update.

        The resulting TTL is the duration until there is 30 hours from
        the time noted by the nextUpdate field in the OCSP response.

        If nextUpdate value is None, an exception is raised.

        :param next_update: datetime value for nextUpdate or None

        :raises .errors.PluginError: If the OCSP response should not be
            configured for use with Apache

        :returns: TTL in seconds.
        :rtype: int

        """
        # hour in seconds
        hour = 3600
        suberror = ""
        if next_update is not None:
            now = datetime.utcnow()
            res_ttl = int((next_update - now).total_seconds())
            if res_ttl > 0:
                safe_ttl = res_ttl - 30 * hour
                if safe_ttl > hour:
                    # Use nextUpdate - 30h if it's over an hour from now
                    return safe_ttl
                else:
                    suberror = ("OCSP response nextUpdate timestamp too "
                                "early: {}. Certbot cannot ensure a safe TTL"
                                "for OCSP staple prefeching.").format(next_update)
            else:
                suberror = ("OCSP response nextUpdate timestamp too "
                            "early: {}").format(next_update)
        else:
            suberror = ("OCSP response nextUpdate not provided with response. "
                        "Staple should not be prefetched.")
        raise errors.PluginError(suberror)

    def _ocsp_response_dbm(self, ocsp_response):
        """Creates a dbm entry for OCSP response data

        :param interfaces.OCSPResponse ocsp_response: Good OCSP response

        :raises .errors.PluginError: If the OCSP response should not be
            configured for use with Apache

        :returns: OCSP response cache data that Apache can use
        :rtype: string

        """
        ttl = self._ocsp_ttl(ocsp_response.next_update)
        response = ocsp_response.bytes
        return apache_util.get_apache_ocsp_struct(ttl, response)

    def _ocsp_prefetch_save(self, cert_path, chain_path):
        """Saves status of current OCSP prefetch, including the last update
        time to determine if an update is needed on later run.

        :param str cert_path: Filesystem path to certificate
        :param str chain_path: Filesystem path to certificate chain file

        """
        status = {
            "lastupdate": time.time(),
            "chain_path": chain_path
        }
        self._ocsp_prefetch[cert_path] = status
        self.storage.put("ocsp_prefetch", self._ocsp_prefetch)
        self.storage.save()

    def _ocsp_prefetch_remove(self, cert_path):
        """Removes OCSP prefetch configuration from PluginStorage object for
        a certificate.

        :param str cert_path: Filesystem path to certificate
        """
        self._ocsp_prefetch.pop(cert_path)
        self.storage.put("ocsp_prefetch", self._ocsp_prefetch)
        self.storage.save()

    def _ocsp_prefetch_fetch_state(self):
        """
        Populates the OCSP prefetch state from the pluginstorage.
        """
        try:
            self._ocsp_prefetch = self.storage.fetch("ocsp_prefetch")
        except KeyError:
            self._ocsp_prefetch = dict()

    def _ocsp_prefetch_backup_db(self):
        """
        Copies the active dbm file to work directory. Logs a debug error
        message if unable to copy, but does not error out as it would
        prevent other critical functions that need to be carried out for
        Apache httpd.

        Erroring out here would prevent any restarts done by Apache plugin.
        """
        self._ensure_ocsp_dirs()
        try:
            apache_util.safe_copy(
                self._ocsp_store,
                self._ocsp_work
            )
        except errors.PluginError:
            logger.debug("Encountered an issue while trying to backup OCSP dbm file")

    def _ocsp_prefetch_restore_db(self):
        """
        Restores the active dbm file from work directory. Logs a debug error
        message if unable to restore, but does not error out as it would
        prevent other critical functions that need to be carried out for
        Apache httpd.

        Erroring out here would prevent any restarts done by Apache plugin.
        """
        self._ensure_ocsp_dirs()
        try:
            filesystem.replace(self._ocsp_work, self._ocsp_store)
        except IOError:
            logger.debug("Encountered an issue when trying to restore OCSP dbm file")

    def enable_ocsp_prefetch(self, lineage, domains, *_unused_args, **_unused_kwargs):
        """Enable OCSP Stapling and prefetching of the responses.

        In OCSP, each client (e.g. browser) would have to query the
        OCSP Responder to validate that the site certificate was not revoked.

        Enabling OCSP Stapling would allow the web-server to query the OCSP
        Responder, and staple its response to the offered certificate during
        TLS. i.e. clients would not have to query the OCSP responder.

        OCSP prefetching functionality addresses some of the pain points in
        the implementation that's currently preset in Apache httpd. The
        mitigation provided by Certbot are:
          * OCSP staples get backed up before, and restored after httpd restart
          * Valid OCSP staples do not get overwritten with errors in case of
            network connectivity or OCSP responder issues
          * The staples get updated asynchronically in the background instead
            of blocking a incoming request.
        """

        # Fail early if we are not able to support this
        self._ensure_ocsp_prefetch_compatibility()
        prefetch_vhosts = set()
        for domain in domains:
            matched_vhosts = self.choose_vhosts(domain, create_if_no_ssl=False)
            # We should be handling only SSL vhosts
            for vh in matched_vhosts:
                if vh.ssl:
                    prefetch_vhosts.add(vh)

        if not prefetch_vhosts:
            raise errors.MisconfigurationError(
                "Could not find VirtualHost to enable OCSP prefetching on."
            )

        try:
            # The try - block is huge, but required for handling rollback properly.
            for vh in prefetch_vhosts:
                self._enable_ocsp_stapling(vh, None, prefetch=True)

            self._ensure_ocsp_dirs()
            self.restart()
            # Ensure Apache has enough time to properly restart and create the file
            time.sleep(2)
            self._ocsp_refresh(lineage.cert_path, lineage.chain_path)
            self._ocsp_prefetch_save(lineage.cert_path, lineage.chain_path)
            self.save("Enabled OCSP prefetching")
        except errors.PluginError as err:
            # Revert the OCSP prefetch configuration
            self.recovery_routine()
            self.restart()
            msg = ("Encountered an error while trying to enable OCSP prefetch "
                   "enhancement: %s\nOCSP prefetch was not enabled.")
            raise errors.PluginError(msg % str(err))

    def deploy_ocsp_prefetch(self, lineage, *_unused_args, **_unused_kwargs):
        """When certificate gets renewed, ensure that we're able to serve an appropriate OCSP
        staple after the restart that replaces the certificate."""

        self._ocsp_prefetch_fetch_state()
        if not self._ocsp_prefetch:
            # No OCSP prefetching enabled for any certificate
            return

        if lineage.cert_path in self._ocsp_prefetch:
            try:
                self._ocsp_try_refresh(lineage.cert_path, lineage.chain_path)
            except OCSPCertificateError:
                # This error was logged and handled already down the stack. Return to avoid save.
                return
            self._ocsp_prefetch_save(lineage.cert_path, lineage.chain_path)

    def update_ocsp_prefetch(self, _unused_lineage, *_unused_args, **_unused_kwargs):
        """Checks all certificates that are managed by OCSP prefetch, and
        refreshes OCSP responses for them if required."""

        self._ocsp_prefetch_fetch_state()
        if not self._ocsp_prefetch:
            # No OCSP prefetching enabled for any certificate
            return

        # make a copy of the list of dictionary keys as we might remove items mid-iteration
        for cert_path in list(self._ocsp_prefetch):
            pf = self._ocsp_prefetch[cert_path]
            if self._ocsp_refresh_needed(pf["lastupdate"]):
                chain_path = pf["chain_path"]
                try:
                    self._ocsp_try_refresh(cert_path, chain_path)
                except OCSPCertificateError:
                    # We want to skip saving in this case, as we just removed the
                    # certificate from prefetch pool.
                    continue
                self._ocsp_prefetch_save(cert_path, chain_path)

    def _ocsp_try_refresh(self, cert_path, chain_path):
        """Attempt to refresh OCSP staple for a certificate.

        :param str cert_path: Path to certificate
        :param str chain_path: Filesystem path to certificate chain file

        :raises OCSPCertificateError: If the given certificate should be
            removed from the OCSP prefetch pool.

        """
        try:
            self._ocsp_refresh(cert_path, chain_path)
        except OCSPCertificateError as err:
            self._ocsp_prefetch_remove(cert_path)
            msg = ("Error when trying to prefetch OCSP staple: {} " +
                    "OCSP prefetch functionality removed for the certificate").format(err)
            logger.warning(msg)
            raise
        except errors.PluginError as err:
            msg = "Encountered a issue when trying to renew OCSP staple: {}".format(err)
            logger.warning(msg)

    def restart(self):
        """Reloads the Apache server. When restarting, Apache deletes
        the DBM cache file used to store OCSP staples. In this override
        function, Certbot checks the pluginstorage if we're supposed to
        manage OCSP prefetching. If needed, Certbot will backup the DBM
        file, restoring it after calling restart.

        :raises .errors.MisconfigurationError: If either the config test
            or reload fails.

        """
        if not self._ocsp_prefetch:
            # Try to populate OCSP prefetch structure from pluginstorage
            self._ocsp_prefetch_fetch_state()
        if self._ocsp_prefetch:
            # OCSP prefetching is enabled, so back up the db
            self._ocsp_prefetch_backup_db()

        try:
            # Ignored because mypy doesn't know that this class is used as
            # a mixin and fails because object has no restart method.
            super(OCSPPrefetchMixin, self).restart()  # type: ignore
        finally:
            if self._ocsp_prefetch:
                # Restore the backed up dbm database
                self._ocsp_prefetch_restore_db()


class OCSPCertificateError(errors.PluginError):
    """Error that prompts for removal of certificate from OCSP prefetch pool."""


OCSPPrefetchEnhancement.register(OCSPPrefetchMixin)  # pylint: disable=no-member