"""A mixin class for OCSP response prefetching for Apache plugin. The OCSP prefetching functionality solves multiple issues in Apache httpd that make using OCSP must-staple error prone. The prefetching functionality works by storing a value to PluginStorage, noting certificates that Certbot should keep OCSP staples (OCSP responses) updated for alongside of the information when the last response was updated by Certbot. When Certbot is invoked, typically by scheduled "certbot renew" and the TTL from "lastupdate" value in PluginStorage entry has expired, Certbot then proceeds to fetch a new OCSP response from the OCSP servers pointed by the certificate. The OCSP response is validated and if valid, stored to Apache DBM cache. A high internal cache expiry value is set for Apache in order to make it to not to discard the stored response and try to renew the staple itself letting Certbot to renew it on its subsequent run instead. The DBM cache file used by Apache is a lightweight key-value storage. For OCSP response caching, the sha1 hash of certificate fingerprint is used as a key. The value consists of expiry time as timestamp in microseconds, \x01 delimiter and the raw OCSP response. When restarting Apache, Certbot backups the current OCSP response cache, and restores it after the restart has happened. This is done because Apache deletes and then recreates the file upon restart. """ from datetime import datetime import logging import time from acme.magic_typing import Dict, Union # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import ocsp from certbot.plugins.enhancements import OCSPPrefetchEnhancement from certbot.compat import filesystem from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import constants logger = logging.getLogger(__name__) class DBMHandler(object): """Context manager to handle DBM file reads and writes""" def __init__(self, filename, mode): self.filename = filename self.filemode = mode self.bsddb = False self.database = None def __enter__(self): """Open the DBM file and return the filehandle""" try: import bsddb self.bsddb = True try: self.database = bsddb.hashopen(self.filename, self.filemode) except Exception: raise errors.PluginError("Unable to open dbm database file.") except ImportError: # Python3 doesn't have bsddb module, so we use dbm.ndbm instead import dbm if self.filename.endswith(".db"): self.filename = self.filename[:-3] try: self.database = dbm.ndbm.open(self.filename, self.filemode) # pylint: disable=no-member except Exception: # This is raised if a file cannot be found raise errors.PluginError("Unable to open dbm database file.") return self.database def __exit__(self, *args): """Close the DBM file""" if self.bsddb: self.database.sync() self.database.close() class OCSPPrefetchMixin(object): """OCSPPrefetchMixin implements OCSP response prefetching""" def __init__(self, *args, **kwargs): self._ocsp_prefetch = {} # type: Dict[str, Dict[str, Union[str, float]]] # This is required because of python super() call chain. # Additionally, mypy isn't able to figure the chain out and needs to be # disabled for this line. See https://github.com/python/mypy/issues/5887 super(OCSPPrefetchMixin, self).__init__(*args, **kwargs) # type: ignore self._ocsp_store = os.path.join(self.config.work_dir, "ocsp", "ocsp_cache.db") self._ocsp_work = os.path.join(self.config.work_dir, "ocsp_work", "ocsp_cache.db") def _ensure_ocsp_dirs(self): """Makes sure that the OCSP directory paths exist.""" for path in [os.path.dirname(self._ocsp_work), os.path.dirname(self._ocsp_store)]: if not os.path.isdir(path): filesystem.makedirs(path, 0o755) def _ensure_ocsp_prefetch_compatibility(self): """Make sure that the operating system supports the required libraries to manage Apache DBM files. :raises: errors.NotSupportedError """ try: import bsddb # pylint: disable=unused-variable except ImportError: import dbm if not hasattr(dbm, 'ndbm') or dbm.ndbm.library != 'Berkeley DB': # pylint: disable=no-member msg = ("Unfortunately your operating system does not have a " "compatible database module available for managing " "Apache OCSP stapling cache database.") raise errors.NotSupportedError(msg) def _ocsp_refresh_needed(self, lastupdate): """Refreshes OCSP response for a certificate if it's due :param int lastupdate: Last update timestamp from pluginstorage entry :returns: If OCSP response was updated :rtype: bool """ ttl = lastupdate + constants.OCSP_INTERNAL_TTL if ttl < time.time(): return True return False def _ocsp_refresh(self, cert_path, chain_path): """Refresh the OCSP response for a certificate :param str cert_path: Filesystem path to certificate file :param str chain_path: Filesystem path to certificate chain file """ self._ensure_ocsp_dirs() try: ocsp_workfile = os.path.join( os.path.dirname(self._ocsp_work), apache_util.certid_sha1_hex(cert_path)) handler = ocsp.RevocationChecker() if not os.path.isfile(cert_path): raise OCSPCertificateError("Certificate has been removed from the system.") if not handler.ocsp_revoked_by_paths(cert_path, chain_path, 10, ocsp_workfile): # Guaranteed good response cert_sha = apache_util.certid_sha1(cert_path) # dbm.open automatically adds the file extension self._write_to_dbm(self._ocsp_store, cert_sha, self._ocsp_response_dbm(ocsp_workfile)) else: logger.warning("Encountered an issue while trying to prefetch OCSP " "response for certificate: %s", cert_path) raise OCSPCertificateError("Certificate has been revoked.") finally: try: os.remove(ocsp_workfile) except OSError: # The OCSP workfile did not exist because of an OCSP response fetching error pass def _write_to_dbm(self, filename, key, value): """Helper method to write an OCSP response cache value to DBM. :param filename: DBM database filename :param bytes key: Database key name :param bytes value: Database entry value """ tmp_file = os.path.join( os.path.dirname(self._ocsp_work), "tmp_" + os.path.basename(filename) ) apache_util.safe_copy(filename, tmp_file) with DBMHandler(tmp_file, 'w') as db: db[key] = value filesystem.replace(tmp_file, filename) def _ocsp_ttl(self, next_update): """Calculates Apache internal TTL for the next OCSP staple update. The resulting TTL is half of the time between now and the time noted by nextUpdate field in OCSP response. If nextUpdate value is None, a default value will be used instead. :param next_update: datetime value for nextUpdate or None :returns: TTL in seconds. :rtype: int """ # hour in seconds hour = 3600 suberror = "" if next_update is not None: now = datetime.utcnow() res_ttl = int((next_update - now).total_seconds()) if res_ttl > 0: safe_ttl = res_ttl - 30 * hour if safe_ttl > hour: # Use nextUpdate - 30h if it's over an hour from now return safe_ttl else: suberror = ("OCSP response nextUpdate timestamp too " "early: {}. Certbot cannot ensure a safe TTL" "for OCSP staple prefeching.").format(next_update) else: suberror = ("OCSP response nextUpdate timestamp too " "early: {}").format(next_update) else: suberror = ("OCSP response nextUpdate not provided with response. " "Staple should not be prefetched.") raise errors.PluginError(suberror) def _ocsp_response_dbm(self, workfile): """Creates a dbm entry for OCSP response data :param str workfile: File path for raw OCSP response :returns: OCSP response cache data that Apache can use :rtype: string """ handler = ocsp.RevocationChecker() _, _, next_update = handler.ocsp_times(workfile) ttl = self._ocsp_ttl(next_update) with open(workfile, 'rb') as fh: response = fh.read() return apache_util.get_apache_ocsp_struct(ttl, response) def _ocsp_prefetch_save(self, cert_path, chain_path): """Saves status of current OCSP prefetch, including the last update time to determine if an update is needed on later run. :param str cert_path: Filesystem path to certificate :param str chain_path: Filesystem path to certificate chain file """ status = { "lastupdate": time.time(), "chain_path": chain_path } self._ocsp_prefetch[cert_path] = status self.storage.put("ocsp_prefetch", self._ocsp_prefetch) self.storage.save() def _ocsp_prefetch_remove(self, cert_path): """Removes OCSP prefetch configuration from PluginStorage object for a certificate. :param str cert_path: Filesystem path to certificate """ self._ocsp_prefetch.pop(cert_path) self.storage.put("ocsp_prefetch", self._ocsp_prefetch) def _ocsp_prefetch_fetch_state(self): """ Populates the OCSP prefetch state from the pluginstorage. """ try: self._ocsp_prefetch = self.storage.fetch("ocsp_prefetch") except KeyError: self._ocsp_prefetch = dict() def _ocsp_prefetch_backup_db(self): """ Copies the active dbm file to work directory. Logs a debug error message if unable to copy, but does not error out as it would prevent other critical functions that need to be carried out for Apache httpd. Erroring out here would prevent any restarts done by Apache plugin. """ self._ensure_ocsp_dirs() try: apache_util.safe_copy( self._ocsp_store, self._ocsp_work ) except errors.PluginError: logger.debug("Encountered an issue while trying to backup OCSP dbm file") def _ocsp_prefetch_restore_db(self): """ Restores the active dbm file from work directory. Logs a debug error message if unable to restore, but does not error out as it would prevent other critical functions that need to be carried out for Apache httpd. Erroring out here would prevent any restarts done by Apache plugin. """ self._ensure_ocsp_dirs() try: filesystem.replace(self._ocsp_work, self._ocsp_store) except IOError: logger.debug("Encountered an issue when trying to restore OCSP dbm file") def enable_ocsp_prefetch(self, lineage, domains): """Enable OCSP Stapling and prefetching of the responses. In OCSP, each client (e.g. browser) would have to query the OCSP Responder to validate that the site certificate was not revoked. Enabling OCSP Stapling would allow the web-server to query the OCSP Responder, and staple its response to the offered certificate during TLS. i.e. clients would not have to query the OCSP responder. OCSP prefetching functionality addresses some of the pain points in the implementation that's currently preset in Apache httpd. The mitigation provided by Certbot are: * OCSP staples get backed up before, and restored after httpd restart * Valid OCSP staples do not get overwritten with errors in case of network connectivity or OCSP responder issues * The staples get updated asynchronically in the background instead of blocking a incoming request. """ # Fail early if we are not able to support this self._ensure_ocsp_prefetch_compatibility() prefetch_vhosts = set() for domain in domains: matched_vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) # We should be handling only SSL vhosts for vh in matched_vhosts: if vh.ssl: prefetch_vhosts.add(vh) if not prefetch_vhosts: raise errors.MisconfigurationError( "Could not find VirtualHost to enable OCSP prefetching on." ) try: # The try - block is huge, but required for handling rollback properly. for vh in prefetch_vhosts: self._enable_ocsp_stapling(vh, None, prefetch=True) self._ensure_ocsp_dirs() self.restart() # Ensure Apache has enough time to properly restart and create the file time.sleep(2) self._ocsp_refresh(lineage.cert_path, lineage.chain_path) self._ocsp_prefetch_save(lineage.cert_path, lineage.chain_path) self.save("Enabled OCSP prefetching") except errors.PluginError as err: # Revert the OCSP prefetch configuration self.recovery_routine() self.restart() msg = ("Encountered an error while trying to enable OCSP prefetch " "enhancement: %s\nOCSP prefetch was not enabled.") raise errors.PluginError(msg % str(err)) def deploy_ocsp_prefetch(self, lineage): """When certificate gets renewed, ensure that we're able to serve an appropriate OCSP staple after the restart that replaces the certificate.""" self._ocsp_prefetch_fetch_state() if not self._ocsp_prefetch: # No OCSP prefetching enabled for any certificate return if lineage.cert_path in self._ocsp_prefetch: pf = self._ocsp_prefetch[lineage.cert_path] try: self._ocsp_try_refresh(lineage.cert_path) except OCSPCertificateError: # This error was logged and handled already down the stack. Return to avoid save. return self._ocsp_prefetch_save(lineage.cert_path, pf["chain_path"]) def update_ocsp_prefetch(self, _unused_lineage): """Checks all certificates that are managed by OCSP prefetch, and refreshes OCSP responses for them if required.""" self._ocsp_prefetch_fetch_state() if not self._ocsp_prefetch: # No OCSP prefetching enabled for any certificate return # make a copy of the list of dictionary keys as we might remove items mid-iteration for cert_path in list(self._ocsp_prefetch): pf = self._ocsp_prefetch[cert_path] if self._ocsp_refresh_needed(pf["lastupdate"]): try: self._ocsp_try_refresh(cert_path) except OCSPCertificateError: # We want to skip saving in this case, as we just removed the # certificate from prefetch pool. continue self._ocsp_prefetch_save(cert_path, pf["chain_path"]) def _ocsp_try_refresh(self, cert_path): """Attempt to refresh OCSP staple for a certificate. :param str cert_path: Path to certificate """ pf = self._ocsp_prefetch[cert_path] try: self._ocsp_refresh(cert_path, pf["chain_path"]) except OCSPCertificateError as err: self._ocsp_prefetch_remove(cert_path) msg = ("Error when trying to prefetch OCSP staple: {} " + "OCSP prefetch functionality removed for the certificate").format(err) logger.warning(msg) raise except errors.PluginError as err: msg = "Encountered a issue when trying to renew OCSP staple: {}".format(err) logger.warning(msg) def restart(self): """Reloads the Apache server. When restarting, Apache deletes the DBM cache file used to store OCSP staples. In this override function, Certbot checks the pluginstorage if we're supposed to manage OCSP prefetching. If needed, Certbot will backup the DBM file, restoring it after calling restart. :raises .errors.MisconfigurationError: If either the config test or reload fails. """ if not self._ocsp_prefetch: # Try to populate OCSP prefetch structure from pluginstorage self._ocsp_prefetch_fetch_state() if self._ocsp_prefetch: # OCSP prefetching is enabled, so back up the db self._ocsp_prefetch_backup_db() try: # Ignored because mypy doesn't know that this class is used as # a mixin and fails because object has no restart method. super(OCSPPrefetchMixin, self).restart() # type: ignore finally: if self._ocsp_prefetch: # Restore the backed up dbm database self._ocsp_prefetch_restore_db() class OCSPCertificateError(errors.PluginError): """Error that prompts for removal of certificate from OCSP prefetch pool.""" OCSPPrefetchEnhancement.register(OCSPPrefetchMixin) # pylint: disable=no-member