diff --git a/.gitignore b/.gitignore index ba843d9cc..1becea3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ letsencrypt.log # auth --cert-path --chain-path /*.pem + +# letstest +tests/letstest/letest-*/ +tests/letstest/*.pem diff --git a/.travis.yml b/.travis.yml index 8dde06ceb..a5d6d8a85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: python services: - rabbitmq - mariadb + # apacheconftest + #- apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml @@ -22,6 +24,9 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=lint - TOXENV=cover +# Disabled for now due to requiring sudo -> causing more boulder integration +# DNS timeouts :( +# - TOXENV=apacheconftest # Only build pushes to the master branch, PRs, and branches beginning with @@ -58,6 +63,12 @@ addons: - openssl # For Boulder integration testing - rsyslog + # for apacheconftest + #- realpath + #- apache2 + #- libapache2-mod-wsgi + #- libapache2-mod-macro + #- sudo install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' diff --git a/Dockerfile b/Dockerfile index 02aa0f0d7..da0110604 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,6 @@ COPY letsencrypt-apache /opt/letsencrypt/src/letsencrypt-apache/ COPY letsencrypt-nginx /opt/letsencrypt/src/letsencrypt-nginx/ -# py26reqs.txt not installed! RUN virtualenv --no-site-packages -p python2 /opt/letsencrypt/venv && \ /opt/letsencrypt/venv/bin/pip install \ -e /opt/letsencrypt/src/acme \ diff --git a/Dockerfile-dev b/Dockerfile-dev index b89411c90..3c5b53966 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -32,7 +32,6 @@ RUN /opt/letsencrypt/src/ubuntu.sh && \ # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -# py26reqs.txt not installed! COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini pep8.travis.sh .pep8 .pylintrc /opt/letsencrypt/src/ # all above files are necessary for setup.py, however, package source diff --git a/MANIFEST.in b/MANIFEST.in index a82c7dd8c..a6f9ae2b6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include py26reqs.txt include README.rst include CHANGES.rst include CONTRIBUTING.md diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index c38cea414..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -1,12 +1,12 @@ """ACME protocol implementation. This module is an implementation of the `ACME protocol`_. Latest -supported version: `v02`_. +supported version: `draft-ietf-acme-01`_. -.. _`ACME protocol`: https://github.com/letsencrypt/acme-spec -.. _`v02`: - https://github.com/letsencrypt/acme-spec/commit/d328fea2d507deb9822793c512830d827a4150c4 +.. _`ACME protocol`: https://ietf-wg-acme.github.io/acme +.. _`draft-ietf-acme-01`: + https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index 7b95e3fce..977a06622 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -226,7 +226,7 @@ class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): :param str name: Name of the field to be encoded. - :raises erors.SerializationError: if field cannot be serialized + :raises errors.SerializationError: if field cannot be serialized :raises errors.Error: if field could not be found """ diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index ab3606efc..600077b20 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -130,7 +130,7 @@ class ImmutableMap(collections.Mapping, collections.Hashable): """Immutable key to value mapping with attribute access.""" __slots__ = () - """Must be overriden in subclasses.""" + """Must be overridden in subclasses.""" def __init__(self, **kwargs): if set(kwargs) != set(self.__slots__): diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 0b73864ec..3b5739da1 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -25,6 +25,8 @@ class Error(jose.JSONObjectWithFields, errors.Error): ('connection', 'The server could not connect to the client to ' 'verify the domain'), ('dnssec', 'The server could not validate a DNSSEC signed domain'), + ('invalidEmail', + 'The provided email for a registration was invalid'), ('malformed', 'The request message was malformed'), ('rateLimited', 'There were too many requests of a given type'), ('serverInternal', 'The server experienced an internal error'), diff --git a/acme/setup.py b/acme/setup.py index e35b40d6e..7314152cd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -10,8 +10,6 @@ install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', - 'ndg-httpsclient', # urllib3 InsecurePlatformWarning (#304) - 'pyasn1', # urllib3 InsecurePlatformWarning (#304) # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15) 'PyOpenSSL>=0.15', 'pyrfc3339', @@ -32,6 +30,11 @@ if sys.version_info < (2, 7): else: install_requires.append('mock') +if sys.version_info < (2, 7, 9): + # For secure SSL connection with Python 2.7 (InsecurePlatformWarning) + install_requires.append('ndg-httpsclient') + install_requires.append('pyasn1') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/bootstrap/README b/bootstrap/README index 89fd8b6ba..d91780903 100644 --- a/bootstrap/README +++ b/bootstrap/README @@ -2,6 +2,5 @@ This directory contains scripts that install necessary OS-specific prerequisite dependencies (see docs/using.rst). General dependencies: -- git-core: py26reqs.txt git+https://* - ca-certificates: communication with demo ACMO server at - https://www.letsencrypt-demo.org, py26reqs.txt git+https://* + https://www.letsencrypt-demo.org diff --git a/bootstrap/_arch_common.sh b/bootstrap/_arch_common.sh index f66067ffb..2b512792f 100755 --- a/bootstrap/_arch_common.sh +++ b/bootstrap/_arch_common.sh @@ -8,7 +8,6 @@ # ./bootstrap/dev/_common_venv.sh deps=" - git python2 python-virtualenv gcc diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 4c6b91a33..8b96fe6f1 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -24,26 +24,56 @@ apt-get update # distro version (#346) virtualenv= -if apt-cache show virtualenv > /dev/null ; then +if apt-cache show virtualenv > /dev/null 2>&1; then virtualenv="virtualenv" fi -if apt-cache show python-virtualenv > /dev/null ; then +if apt-cache show python-virtualenv > /dev/null 2>&1; then virtualenv="$virtualenv python-virtualenv" fi +augeas_pkg=libaugeas0 +AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + +if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then + /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." + sleep 1s + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + + echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list + apt-get update + fi + fi + apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 + augeas_pkg= + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs +fi + apt-get install -y --no-install-recommends \ - git \ python \ python-dev \ $virtualenv \ gcc \ dialog \ - libaugeas0 \ + $augeas_pkg \ libssl-dev \ libffi-dev \ ca-certificates \ + + if ! command -v virtualenv > /dev/null ; then echo Failed to install a working \"virtualenv\" command, exiting exit 1 diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh index a718db7ff..f49dc00f0 100755 --- a/bootstrap/_gentoo_common.sh +++ b/bootstrap/_gentoo_common.sh @@ -1,6 +1,6 @@ #!/bin/sh -PACKAGES="dev-vcs/git +PACKAGES=" dev-lang/python:2.7 dev-python/virtualenv dev-util/dialog diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh index 411d7bd92..db1665268 100755 --- a/bootstrap/_rpm_common.sh +++ b/bootstrap/_rpm_common.sh @@ -33,9 +33,7 @@ then fi fi -# "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails) if ! $tool install -y \ - git-core \ gcc \ dialog \ augeas-libs \ @@ -47,3 +45,11 @@ then echo "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi + + +if $tool list installed "httpd" >/dev/null 2>&1; then + if ! $tool install -y mod_ssl + then + echo "Apache found, but mod_ssl could not be installed." + fi +fi diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh index 46f9d693b..efeebe4f8 100755 --- a/bootstrap/_suse_common.sh +++ b/bootstrap/_suse_common.sh @@ -2,7 +2,7 @@ # SLE12 don't have python-virtualenv -zypper -nq in -l git-core \ +zypper -nq in -l \ python \ python-devel \ python-virtualenv \ diff --git a/bootstrap/dev/venv.sh b/bootstrap/dev/venv.sh index 2bd32a89b..11ab417dd 100755 --- a/bootstrap/dev/venv.sh +++ b/bootstrap/dev/venv.sh @@ -4,7 +4,6 @@ export VENV_ARGS="--python python2" ./bootstrap/dev/_venv_common.sh \ - -r py26reqs.txt \ -e acme[testing] \ -e .[dev,docs,testing] \ -e letsencrypt-apache \ diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh index 180ee21b4..4482c35cd 100755 --- a/bootstrap/freebsd.sh +++ b/bootstrap/freebsd.sh @@ -1,7 +1,6 @@ #!/bin/sh -xe pkg install -Ay \ - git \ python \ py27-virtualenv \ augeas \ diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh index ff1a50c6c..5042178d9 100755 --- a/bootstrap/venv.sh +++ b/bootstrap/venv.sh @@ -20,7 +20,7 @@ fi pip install -U setuptools pip install -U pip -pip install -U -r py26reqs.txt letsencrypt letsencrypt-apache # letsencrypt-nginx +pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx echo echo "Congratulations, Let's Encrypt has been successfully installed/updated!" diff --git a/docs/contributing.rst b/docs/contributing.rst index c71aefeec..6c70830b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -65,8 +65,14 @@ Testing The following tools are there to help you: -- ``tox`` starts a full set of tests. Please make sure you run it - before submitting a new pull request. +- ``tox`` starts a full set of tests. Please note that it includes + apacheconftest, which uses the system's Apache install to test config file + parsing, so it should only be run on systems that have an + experimental, non-production Apache2 install on them. ``tox -e + apacheconftest`` can be used to run those specific Apache conf tests. + +- ``tox -e py27``, ``tox -e py26`` etc, run unit tests for specific Python + versions. - ``tox -e cover`` checks the test coverage only. Calling the ``./tox.cover.sh`` script directly (or even ``./tox.cover.sh $pkg1 diff --git a/docs/using.rst b/docs/using.rst index 1423d6eba..5da13f02c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -64,7 +64,7 @@ or for full help, type: ``letsencrypt-auto`` is the recommended method of running the Let's Encrypt client beta releases on systems that don't have a packaged version. Debian, -Arch linux and FreeBSD now have native packages, so on those +Arch linux, FreeBSD, and OpenBSD now have native packages, so on those systems you can just install ``letsencrypt`` (and perhaps ``letsencrypt-apache``). If you'd like to run the latest copy from Git, or run your own locally modified copy of the client, follow the instructions in @@ -351,6 +351,11 @@ Operating System Packages * Port: ``cd /usr/ports/security/py-letsencrypt && make install clean`` * Package: ``pkg install py27-letsencrypt`` +**OpenBSD** + + * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` + * Package: ``pkg_add letsencrypt`` + **Arch Linux** .. code-block:: shell @@ -366,7 +371,7 @@ If you run Debian Stretch or Debian Sid, you can install letsencrypt packages. sudo apt-get update sudo apt-get install letsencrypt python-letsencrypt-apache -If you don't want to use the Apache plugin, you can ommit the +If you don't want to use the Apache plugin, you can omit the ``python-letsencrypt-apache`` package. Packages for Debian Jessie are coming in the next few weeks. diff --git a/examples/cli.ini b/examples/cli.ini index 6b6b05d7d..f0c993c57 100644 --- a/examples/cli.ini +++ b/examples/cli.ini @@ -5,9 +5,6 @@ # Use a 4096 bit RSA key instead of 2048 rsa-key-size = 4096 -# Always use the staging/testing server -server = https://acme-staging.api.letsencrypt.org/directory - # Uncomment and update to register with the specified e-mail address # email = foo@example.com diff --git a/examples/dev-cli.ini b/examples/dev-cli.ini index be703814a..c02038ca1 100644 --- a/examples/dev-cli.ini +++ b/examples/dev-cli.ini @@ -1,3 +1,6 @@ +# Always use the staging/testing server - avoids rate limiting +server = https://acme-staging.api.letsencrypt.org/directory + # This is an example configuration file for developers config-dir = /tmp/le/conf work-dir = /tmp/le/conf diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index d665ea7a7..0f2cb7b45 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -59,7 +59,7 @@ let empty = Util.empty_dos let indent = Util.indent (* borrowed from shellvars.aug *) -let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ '"\t\r\n])|\\\\"|\\\\'/ +let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 3a0b90fc0..d936309ae 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -86,18 +86,25 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @classmethod def add_parser_arguments(cls, add): - add("ctl", default=constants.CLI_DEFAULTS["ctl"], - help="Path to the 'apache2ctl' binary, used for 'configtest', " - "retrieving the Apache2 version number, and initialization " - "parameters.") - add("enmod", default=constants.CLI_DEFAULTS["enmod"], + add("enmod", default=constants.os_constant("enmod"), help="Path to the Apache 'a2enmod' binary.") - add("dismod", default=constants.CLI_DEFAULTS["dismod"], + add("dismod", default=constants.os_constant("dismod"), help="Path to the Apache 'a2dismod' binary.") - add("le-vhost-ext", default=constants.CLI_DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=constants.os_constant("le_vhost_ext"), help="SSL vhost configuration extension.") - add("server-root", default=constants.CLI_DEFAULTS["server_root"], + add("server-root", default=constants.os_constant("server_root"), help="Apache server root directory.") + add("vhost-root", default=constants.os_constant("vhost_root"), + help="Apache server VirtualHost configuration root") + add("challenge-location", + default=constants.os_constant("challenge_location"), + help="Directory path for challenge configuration.") + add("handle-modules", default=constants.os_constant("handle_mods"), + help="Let installer handle enabling required modules for you." + + "(Only Ubuntu/Debian currently)") + add("handle-sites", default=constants.os_constant("handle_sites"), + help="Let installer handle enabling sites for you." + + "(Only Ubuntu/Debian currently)") le_util.add_deprecated_argument(add, "init-script", 1) def __init__(self, *args, **kwargs): @@ -137,18 +144,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Verify Apache is installed - for exe in (self.conf("ctl"), self.conf("enmod"), self.conf("dismod")): - if not le_util.exe_exists(exe): - raise errors.NoInstallationError + if not le_util.exe_exists(constants.os_constant("restart_cmd")[0]): + raise errors.NoInstallationError # Make sure configuration is valid self.config_test() - self.parser = parser.ApacheParser( - self.aug, self.conf("server-root"), self.conf("ctl")) - # Check for errors in parsing files with Augeas - self.check_parsing_errors("httpd.aug") - # Set Version if self.version is None: self.version = self.get_version() @@ -156,6 +157,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) + self.parser = parser.ApacheParser( + self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.version) + # Check for errors in parsing files with Augeas + self.check_parsing_errors("httpd.aug") + # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() @@ -236,9 +243,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if chain_path is not None: self.save_notes += "\tSSLCertificateChainFile %s\n" % chain_path - # Make sure vhost is enabled - if not vhost.enabled: - self.enable_site(vhost) + # Make sure vhost is enabled if distro with enabled / available + if self.conf("handle-sites"): + if not vhost.enabled: + self.enable_site(vhost) def choose_vhost(self, target_name, temp=False): """Chooses a virtual host based on the given domain name. @@ -458,7 +466,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): is_ssl = True filename = get_file_path(path) - is_enabled = self.is_site_enabled(filename) + if self.conf("handle-sites"): + is_enabled = self.is_site_enabled(filename) + else: + is_enabled = True macro = False if "/macro/" in path.lower(): @@ -469,7 +480,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._add_servernames(vhost) return vhost - # TODO: make "sites-available" a configurable directory def get_virtual_hosts(self): """Returns list of virtual hosts found in the Apache configuration. @@ -478,10 +488,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: list """ - # Search sites-available, httpd.conf for possible virtual hosts + # Search vhost-root, httpd.conf for possible virtual hosts paths = self.aug.match( - ("/files%s/sites-available//*[label()=~regexp('%s')]" % - (self.parser.root, parser.case_i("VirtualHost")))) + ("/files%s//*[label()=~regexp('%s')]" % + (self.conf("vhost-root"), parser.case_i("VirtualHost")))) vhs = [] @@ -540,15 +550,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str port: Port to listen on """ - if "ssl_module" not in self.parser.modules: - self.enable_mod("ssl", temp=temp) + self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] # In case no Listens are set (which really is a broken apache config) if not listens: listens = ["80"] + if port in listens: + return for listen in listens: # For any listen statement, check if the machine also listens on Port 443. # If not, add such a listen statement. @@ -583,6 +594,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ip, port, self.parser.loc["listen"]) listens.append("%s:%s" % (ip, port)) + def prepare_https_modules(self, temp): + """Helper method for prepare_server_https, taking care of enabling + needed modules + + :param boolean temp: If the change is temporary + """ + + if self.conf("handle-modules"): + if "ssl_module" not in self.parser.modules: + self.enable_mod("ssl", temp=temp) + if self.version >= (2, 4) and ("socache_shmcb_module" not in + self.parser.modules): + self.enable_mod("socache_shmcb", temp=temp) + def make_addrs_sni_ready(self, addrs): """Checks to see if the server is ready for SNI challenges. @@ -605,7 +630,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``letsencrypt_apache.constants.CLI_DEFAULTS["le_vhost_ext"]`` + ``letsencrypt_apache.constants.os_constant("le_vhost_ext")`` .. note:: This function saves the configuration @@ -1067,8 +1092,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join( - self.parser.root, "sites-available", redirect_filename) + redirect_filepath = os.path.join(self.conf("vhost-root"), redirect_filename) # Register the new file that will be created # Note: always register the creation before writing to ensure file will @@ -1154,7 +1178,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: bool """ + enabled_dir = os.path.join(self.parser.root, "sites-enabled") + if not os.path.isdir(enabled_dir): + error_msg = ("Directory '{0}' does not exist. Please ensure " + "that the values for --apache-handle-sites and " + "--apache-server-root are correct for your " + "environment.".format(enabled_dir)) + raise errors.ConfigurationError(error_msg) for entry in os.listdir(enabled_dir): try: if filecmp.cmp(avail_fp, os.path.join(enabled_dir, entry)): @@ -1242,7 +1273,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Modules can enable additional config files. Variables may be defined # within these new configuration sections. # Reload is not necessary as DUMP_RUN_CFG uses latest config. - self.parser.update_runtime_variables(self.conf("ctl")) + self.parser.update_runtime_variables() def _add_parser_mod(self, mod_name): """Shortcut for updating parser modules.""" @@ -1281,7 +1312,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script([self.conf("ctl"), "-k", "graceful"]) + le_util.run_script(constants.os_constant("restart_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1292,7 +1323,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script([self.conf("ctl"), "configtest"]) + le_util.run_script(constants.os_constant("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1308,10 +1339,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - stdout, _ = le_util.run_script([self.conf("ctl"), "-v"]) + stdout, _ = le_util.run_script( + constants.os_constant("version_cmd")) except errors.SubprocessError: raise errors.PluginError( - "Unable to run %s -v" % self.conf("ctl")) + "Unable to run %s -v" % + constants.os_constant("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) @@ -1400,7 +1433,7 @@ def _get_mod_deps(mod_name): """ deps = { - "ssl": ["setenvif", "mime", "socache_shmcb"] + "ssl": ["setenvif", "mime"] } return deps.get(mod_name, []) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index eb004b975..8ac88b197 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -1,14 +1,62 @@ """Apache plugin constants.""" import pkg_resources +from letsencrypt import le_util -CLI_DEFAULTS = dict( +CLI_DEFAULTS_DEBIAN = dict( server_root="/etc/apache2", - ctl="apache2ctl", + vhost_root="/etc/apache2/sites-available", + vhost_files="*", + version_cmd=['apache2ctl', '-v'], + define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", + handle_mods=True, + handle_sites=True, + challenge_location="/etc/apache2" ) +CLI_DEFAULTS_CENTOS = dict( + server_root="/etc/httpd", + vhost_root="/etc/httpd/conf.d", + vhost_files="*.conf", + version_cmd=['apachectl', '-v'], + define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/httpd/conf.d" +) +CLI_DEFAULTS_GENTOO = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/vhosts.d", + vhost_files="*.conf", + version_cmd=['/usr/sbin/apache2', '-v'], + define_cmd=['/usr/sbin/apache2', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/apache2/vhosts.d" +) +CLI_DEFAULTS = { + "debian": CLI_DEFAULTS_DEBIAN, + "ubuntu": CLI_DEFAULTS_DEBIAN, + "centos": CLI_DEFAULTS_CENTOS, + "centos linux": CLI_DEFAULTS_CENTOS, + "fedora": CLI_DEFAULTS_CENTOS, + "red hat enterprise linux server": CLI_DEFAULTS_CENTOS, + "gentoo base system": CLI_DEFAULTS_GENTOO +} """CLI defaults.""" MOD_SSL_CONF_DEST = "options-ssl-apache.conf" @@ -33,7 +81,7 @@ REWRITE_HTTPS_ARGS_WITH_END = [ https vhost""" HSTS_ARGS = ["always", "set", "Strict-Transport-Security", - "\"max-age=31536000; includeSubDomains\""] + "\"max-age=31536000\""] """Apache header arguments for HSTS""" UIR_ARGS = ["always", "set", "Content-Security-Policy", @@ -42,3 +90,15 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy", HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS, "Upgrade-Insecure-Requests": UIR_ARGS} + +def os_constant(key): + """Get a constant value for operating system + :param key: name of cli constant + :return: value of constant for active os + """ + os_info = le_util.get_os_info() + try: + constants = CLI_DEFAULTS[os_info[0].lower()] + except KeyError: + constants = CLI_DEFAULTS["debian"] + return constants[key] diff --git a/letsencrypt-apache/letsencrypt_apache/options-ssl-apache.conf b/letsencrypt-apache/letsencrypt_apache/options-ssl-apache.conf index 2a724d7ec..ec07a4ba3 100644 --- a/letsencrypt-apache/letsencrypt_apache/options-ssl-apache.conf +++ b/letsencrypt-apache/letsencrypt_apache/options-ssl-apache.conf @@ -14,9 +14,9 @@ SSLOptions +StrictRequire LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common -CustomLog /var/log/apache2/access.log vhost_combined -LogLevel warn -ErrorLog /var/log/apache2/error.log +#CustomLog /var/log/apache2/access.log vhost_combined +#LogLevel warn +#ErrorLog /var/log/apache2/error.log # Always ensure Cookies have "Secure" set (JAH 2012/1) #Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4" diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index ec5211ae4..593c807cc 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -8,6 +8,7 @@ import subprocess from letsencrypt import errors +from letsencrypt_apache import constants logger = logging.getLogger(__name__) @@ -19,7 +20,6 @@ class ApacheParser(object): :ivar str root: Normalized absolute path to the server root directory. Without trailing slash. - :ivar str root: Server root :ivar set modules: All module names that are currently enabled. :ivar dict loc: Location to place directives, root - configuration origin, default - user config file, name - NameVirtualHost, @@ -28,7 +28,7 @@ class ApacheParser(object): arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") fnmatch_chars = set(["*", "?", "\\", "[", "]"]) - def __init__(self, aug, root, ctl): + def __init__(self, aug, root, vhostroot, version=(2, 4)): # Note: Order is important here. # This uses the binary, so it can be done first. @@ -36,7 +36,8 @@ class ApacheParser(object): # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine # This only handles invocation parameters and Define directives! self.variables = {} - self.update_runtime_variables(ctl) + if version >= (2, 4): + self.update_runtime_variables() self.aug = aug # Find configuration root and make sure augeas can parse it. @@ -44,6 +45,8 @@ class ApacheParser(object): self.loc = {"root": self._find_config_root()} self._parse_file(self.loc["root"]) + self.vhostroot = os.path.abspath(vhostroot) + # This problem has been fixed in Augeas 1.0 self.standardize_excl() @@ -56,9 +59,14 @@ class ApacheParser(object): # Set up rest of locations self.loc.update(self._set_locations()) - # Must also attempt to parse sites-available or equivalent - # Sites-available is not included naturally in configuration - self._parse_file(os.path.join(self.root, "sites-available") + "/*") + # Must also attempt to parse virtual host root + self._parse_file(self.vhostroot + "/" + + constants.os_constant("vhost_files")) + + # check to see if there were unparsed define statements + if version < (2, 4): + if self.find_dir("Define", exclude=False): + raise errors.PluginError("Error parsing runtime variables") def init_modules(self): """Iterates on the configuration until no new modules are loaded. @@ -84,7 +92,7 @@ class ApacheParser(object): self.modules.add( os.path.basename(self.get_arg(match_filename))[:-2] + "c") - def update_runtime_variables(self, ctl): + def update_runtime_variables(self): """" .. note:: Compile time variables (apache2ctl -V) are not used within the @@ -94,19 +102,19 @@ class ApacheParser(object): .. todo:: Create separate compile time variables... simply for arg_get() """ - stdout = self._get_runtime_cfg(ctl) + stdout = self._get_runtime_cfg() variables = dict() matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) try: matches.remove("DUMP_RUN_CFG") except ValueError: - raise errors.PluginError("Unable to parse runtime variables") + return for match in matches: if match.count("=") > 1: logger.error("Unexpected number of equal signs in " - "apache2ctl -D DUMP_RUN_CFG") + "runtime config dump.") raise errors.PluginError( "Error parsing Apache runtime variables") parts = match.partition("=") @@ -114,7 +122,7 @@ class ApacheParser(object): self.variables = variables - def _get_runtime_cfg(self, ctl): # pylint: disable=no-self-use + def _get_runtime_cfg(self): # pylint: disable=no-self-use """Get runtime configuration info. :returns: stdout from DUMP_RUN_CFG @@ -122,16 +130,18 @@ class ApacheParser(object): """ try: proc = subprocess.Popen( - [ctl, "-t", "-D", "DUMP_RUN_CFG"], + constants.os_constant("define_cmd"), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() except (OSError, ValueError): logger.error( - "Error accessing %s for runtime parameters!%s", ctl, os.linesep) + "Error running command %s for runtime parameters!%s", + constants.os_constant("define_cmd"), os.linesep) raise errors.MisconfigurationError( - "Error accessing loaded Apache parameters: %s", ctl) + "Error accessing loaded Apache parameters: %s", + constants.os_constant("define_cmd")) # Small errors that do not impede if proc.returncode != 0: logger.warn("Error in checking parameter list: %s", stderr) @@ -546,8 +556,7 @@ class ApacheParser(object): def _find_config_root(self): """Find the Apache Configuration Root file.""" - location = ["apache2.conf", "httpd.conf"] - + location = ["apache2.conf", "httpd.conf", "conf/httpd.conf"] for name in location: if os.path.isfile(os.path.join(self.root, name)): return os.path.join(self.root, name) diff --git a/tests/apache-conf-files/NEEDED.txt b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/NEEDED.txt similarity index 100% rename from tests/apache-conf-files/NEEDED.txt rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/NEEDED.txt diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test new file mode 100755 index 000000000..4e0443bb7 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -0,0 +1,78 @@ +#!/bin/bash + +# A hackish script to see if the client is behaving as expected +# with each of the "passing" conf files. + +export EA=/etc/apache2/ +TESTDIR="`dirname $0`" +LEROOT="`realpath \"$TESTDIR/../../../../\"`" +cd $TESTDIR/passing +LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" + +function CleanupExit() { + echo control c, exiting tests... + if [ "$f" != "" ] ; then + Cleanup + fi + exit 1 +} + +function Setup() { + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo cp "$f" "$EA"/sites-available/ + sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + sudo echo """ + + ServerName example.com + DocumentRoot /tmp/ + ErrorLog /tmp/error.log + CustomLog /tmp/requests.log combined +""" >> $EA/sites-available/throwaway-example.conf + else + TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" + sudo cp -a "$APPEND_APACHECONF" "$TMP" + sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" + fi +} + +function Cleanup() { + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo rm /etc/apache2/sites-{enabled,available}/"$f" + sudo rm $EA/sites-available/throwaway-example.conf + else + sudo mv "$TMP" "$APPEND_APACHECONF" + fi +} + +# if our environment asks us to enable modules, do our best! +if [ "$1" = --debian-modules ] ; then + sudo apt-get install -y libapache2-mod-wsgi + sudo apt-get install -y libapache2-mod-macro + + for mod in ssl rewrite macro wsgi deflate userdir version mime ; do + sudo a2enmod $mod + done +fi + + +FAILS=0 +trap CleanupExit INT +for f in *.conf ; do + echo -n testing "$f"... + Setup + RESULT=`echo c | sudo "$LETSENCRYPT" -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then + echo passed + else + echo failed + echo $RESULT + echo + echo + FAILS=`expr $FAILS + 1` + fi + Cleanup +done +if [ "$FAILS" -ne 0 ] ; then + exit 1 +fi +exit 0 diff --git a/tests/apache-conf-files/failing/ipv6-1143.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143.conf similarity index 100% rename from tests/apache-conf-files/failing/ipv6-1143.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143.conf diff --git a/tests/apache-conf-files/failing/ipv6-1143b.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143b.conf similarity index 100% rename from tests/apache-conf-files/failing/ipv6-1143b.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143b.conf diff --git a/tests/apache-conf-files/failing/missing-double-quote-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/missing-double-quote-1724.conf similarity index 100% rename from tests/apache-conf-files/failing/missing-double-quote-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/missing-double-quote-1724.conf diff --git a/tests/apache-conf-files/failing/multivhost-1093.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093.conf similarity index 100% rename from tests/apache-conf-files/failing/multivhost-1093.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093.conf diff --git a/tests/apache-conf-files/failing/multivhost-1093b.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093b.conf similarity index 100% rename from tests/apache-conf-files/failing/multivhost-1093b.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093b.conf diff --git a/tests/apache-conf-files/passing/1626-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/1626-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/1626-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/1626-1531.conf diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules new file mode 100644 index 000000000..32c3ef019 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules @@ -0,0 +1,6 @@ +# Modules required to parse these conf files: +ssl +rewrite +macro +wsgi +deflate diff --git a/tests/apache-conf-files/passing/anarcat-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/anarcat-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/anarcat-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/anarcat-1531.conf diff --git a/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf similarity index 100% rename from tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf diff --git a/tests/apache-conf-files/passing/drupal-htaccess-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-htaccess-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/drupal-htaccess-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-htaccess-1531.conf diff --git a/tests/apache-conf-files/passing/example-1755.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-1755.conf similarity index 100% rename from tests/apache-conf-files/passing/example-1755.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-1755.conf diff --git a/tests/apache-conf-files/passing/example-ssl.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-ssl.conf similarity index 100% rename from tests/apache-conf-files/passing/example-ssl.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-ssl.conf diff --git a/tests/apache-conf-files/passing/example.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example.conf similarity index 100% rename from tests/apache-conf-files/passing/example.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example.conf diff --git a/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt similarity index 100% rename from tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt diff --git a/tests/apache-conf-files/passing/finalize-1243.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.conf similarity index 100% rename from tests/apache-conf-files/passing/finalize-1243.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.conf diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf new file mode 100644 index 000000000..f257dd9a8 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf @@ -0,0 +1,21 @@ + + + WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=www-data group=www-data + WSGIProcessGroup _graphite + WSGIImportScript /usr/share/graphite-web/graphite.wsgi process-group=_graphite application-group=%{GLOBAL} + WSGIScriptAlias / /usr/share/graphite-web/graphite.wsgi + + Alias /content/ /usr/share/graphite-web/static/ + + SetHandler None + + + ErrorLog ${APACHE_LOG_DIR}/graphite-web_error.log + + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/graphite-web_access.log combined + + diff --git a/tests/apache-conf-files/passing/missing-quote-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/missing-quote-1724.conf similarity index 100% rename from tests/apache-conf-files/passing/missing-quote-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/missing-quote-1724.conf diff --git a/tests/apache-conf-files/passing/modmacro-1385.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/modmacro-1385.conf similarity index 100% rename from tests/apache-conf-files/passing/modmacro-1385.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/modmacro-1385.conf diff --git a/tests/apache-conf-files/passing/owncloud-1264.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/owncloud-1264.conf similarity index 100% rename from tests/apache-conf-files/passing/owncloud-1264.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/owncloud-1264.conf diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf new file mode 100644 index 000000000..26214e7b0 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf @@ -0,0 +1,7 @@ + + RewriteEngine On + RewriteCond %{REQUEST_URI} ^.*(,|;|:|<|>|">|"<|/|\\\.\.\\).* [NC,OR] + RewriteCond %{REQUEST_URI} ^.*(\=|\@|\[|\]|\^|\`|\{|\}|\~).* [NC,OR] + RewriteCond %{REQUEST_URI} ^.*(\'|%0A|%0D|%27|%3C|%3E|%00).* [NC] + RewriteRule ^(.*)$ - [F,L] + diff --git a/tests/apache-conf-files/passing/roundcube-1222.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/roundcube-1222.conf similarity index 100% rename from tests/apache-conf-files/passing/roundcube-1222.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/roundcube-1222.conf diff --git a/tests/apache-conf-files/passing/semacode-1598.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/semacode-1598.conf similarity index 100% rename from tests/apache-conf-files/passing/semacode-1598.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/semacode-1598.conf diff --git a/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess similarity index 100% rename from tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess diff --git a/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf similarity index 100% rename from tests/apache-conf-files/passing/two-blocks-one-line-1693.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf diff --git a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py index 815e6fc44..b70e1c7f1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py @@ -17,7 +17,7 @@ class AugeasConfiguratorTest(util.ApacheTest): super(AugeasConfiguratorTest, self).setUp() self.config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir) + self.config_path, self.vhost_path, self.config_dir, self.work_dir) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/two_vhost_80") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index f2bf89d2c..9838b4f52 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -27,11 +27,22 @@ class TwoVhost80Test(util.ApacheTest): super(TwoVhost80Test, self).setUp() self.config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir) - + self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.config = self.mock_deploy_cert(self.config) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/two_vhost_80") + def mock_deploy_cert(self, config): + """A test for a mock deploy cert""" + self.config.real_deploy_cert = self.config.deploy_cert + def mocked_deploy_cert(*args, **kwargs): + """a helper to mock a deployed cert""" + with mock.patch( + "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + config.real_deploy_cert(*args, **kwargs) + self.config.deploy_cert = mocked_deploy_cert + return self.config + def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) @@ -116,6 +127,24 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(found, 6) + # Handle case of non-debian layout get_virtual_hosts + orig_conf = self.config.conf + with mock.patch( + "letsencrypt_apache.configurator.ApacheConfigurator.conf" + ) as mock_conf: + def conf_sideeffect(key): + """Handle calls to configurator.conf() + :param key: configuration key + :return: configuration value + """ + if key == "handle-sites": + return False + else: + return orig_conf(key) + mock_conf.side_effect = conf_sideeffect + vhs = self.config.get_virtual_hosts() + self.assertEqual(len(vhs), 6) + @mock.patch("letsencrypt_apache.display_ops.select_vhost") def test_choose_vhost_none_avail(self, mock_select): mock_select.return_value = None @@ -201,6 +230,11 @@ class TwoVhost80Test(util.ApacheTest): self.assertFalse(self.config.is_site_enabled(self.vh_truth[1].filep)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].filep)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].filep)) + with mock.patch("os.path.isdir") as mock_isdir: + mock_isdir.return_value = False + self.assertRaises(errors.ConfigurationError, + self.config.is_site_enabled, + "irrelevant") @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -244,13 +278,14 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl(self): self.config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] + self.config = self.mock_deploy_cert(self.config) self.config.deploy_cert( "random.demo", "example/cert.pem", "example/key.pem", "example/cert_chain.pem", "example/fullchain.pem") @@ -276,7 +311,8 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl_no_fullchain(self): self.config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -289,7 +325,8 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_old_apache_no_chain(self): self.config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir, version=(2, 4, 7)) + self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7)) + self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -424,6 +461,25 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(mock_add_dir.call_args_list[1][0][2], ["[::1]:8080", "https"]) self.assertEqual(mock_add_dir.call_args_list[2][0][2], ["1.1.1.1:8080", "https"]) + def test_prepare_server_https_mixed_listen(self): + + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:8080", "443"] + mock_add_dir = mock.Mock() + mock_enable = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir_to_ifmodssl = mock_add_dir + self.config.enable_mod = mock_enable + + # Test Listen statements with specific ip listeed + self.config.prepare_server_https("443") + # Should only be 2 here, as the third interface already listens to the correct port + self.assertEqual(mock_add_dir.call_count, 0) + def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py new file mode 100644 index 000000000..289b61bb1 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py @@ -0,0 +1,27 @@ +"""Test for letsencrypt_apache.configurator.""" + +import mock +import unittest + +from letsencrypt_apache import constants + + +class ConstantsTest(unittest.TestCase): + + @mock.patch("letsencrypt.le_util.get_os_info") + def test_get_debian_value(self, os_info): + os_info.return_value = ('Debian', '', '') + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/apache2/sites-available") + + @mock.patch("letsencrypt.le_util.get_os_info") + def test_get_centos_value(self, os_info): + os_info.return_value = ('CentOS Linux', '', '') + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/httpd/conf.d") + + @mock.patch("letsencrypt.le_util.get_os_info") + def test_get_default_value(self, os_info): + os_info.return_value = ('Nonexistent Linux', '', '') + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/apache2/sites-available") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index bc1f316f9..b871f89b7 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -145,25 +145,26 @@ class BasicParserTest(util.ParserTest): expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.update_runtime_variables("ctl") + self.parser.update_runtime_variables() self.assertEqual(self.parser.variables, expected_vars) @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_vars_bad_output(self, mock_cfg): mock_cfg.return_value = "Define: TLS=443=24" - self.assertRaises( - errors.PluginError, self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables() mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24" self.assertRaises( - errors.PluginError, self.parser.update_runtime_variables, "ctl") + errors.PluginError, self.parser.update_runtime_variables) + @mock.patch("letsencrypt_apache.constants.os_constant") @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen): + def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): mock_popen.side_effect = OSError + mock_const.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, - self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables) @mock.patch("letsencrypt_apache.parser.subprocess.Popen") def test_update_runtime_vars_bad_exit(self, mock_popen): @@ -171,7 +172,7 @@ class BasicParserTest(util.ParserTest): mock_popen.returncode = -1 self.assertRaises( errors.MisconfigurationError, - self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables) class ParserInitTest(util.ApacheTest): @@ -185,6 +186,15 @@ class ParserInitTest(util.ApacheTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") + def test_unparsable(self, mock_cfg): + from letsencrypt_apache.parser import ApacheParser + mock_cfg.return_value = ('Define: TEST') + self.assertRaises( + errors.PluginError, + ApacheParser, self.aug, os.path.relpath(self.config_path), + "/dummy/vhostpath", version=(2, 2, 22)) + def test_root_normalized(self): from letsencrypt_apache.parser import ApacheParser @@ -193,7 +203,9 @@ class ParserInitTest(util.ApacheTest): path = os.path.join( self.temp_dir, "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2") - parser = ApacheParser(self.aug, path, "dummy_ctl") + + parser = ApacheParser(self.aug, path, + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) @@ -202,7 +214,8 @@ class ParserInitTest(util.ApacheTest): with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): parser = ApacheParser( - self.aug, os.path.relpath(self.config_path), "dummy_ctl") + self.aug, os.path.relpath(self.config_path), + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) @@ -211,7 +224,8 @@ class ParserInitTest(util.ApacheTest): with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): parser = ApacheParser( - self.aug, self.config_path + os.path.sep, "dummy_ctl") + self.aug, self.config_path + os.path.sep, + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py index f4dff7734..b69406932 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py @@ -20,7 +20,7 @@ class TlsSniPerformTest(util.ApacheTest): super(TlsSniPerformTest, self).setUp() config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir) + self.config_path, self.vhost_path, self.config_dir, self.work_dir) config.config.tls_sni_01_port = 443 from letsencrypt_apache import tls_sni_01 @@ -78,7 +78,9 @@ class TlsSniPerformTest(util.ApacheTest): # pylint: disable=protected-access self.sni._setup_challenge_cert = mock_setup_cert - sni_responses = self.sni.perform() + with mock.patch( + "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + sni_responses = self.sni.perform() self.assertEqual(mock_setup_cert.call_count, 2) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 0c60373f2..798d4814b 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -23,7 +23,8 @@ from letsencrypt_apache import obj class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods def setUp(self, test_dir="debian_apache_2_4/two_vhost_80", - config_root="debian_apache_2_4/two_vhost_80/apache2"): + config_root="debian_apache_2_4/two_vhost_80/apache2", + vhost_root="debian_apache_2_4/two_vhost_80/apache2/sites-available"): # pylint: disable=arguments-differ super(ApacheTest, self).setUp() @@ -36,6 +37,7 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods constants.MOD_SSL_CONF_DEST) self.config_path = os.path.join(self.temp_dir, config_root) + self.vhost_path = os.path.join(self.temp_dir, vhost_root) self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) @@ -44,8 +46,9 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods def setUp(self, test_dir="debian_apache_2_4/two_vhost_80", - config_root="debian_apache_2_4/two_vhost_80/apache2"): - super(ParserTest, self).setUp(test_dir, config_root) + config_root="debian_apache_2_4/two_vhost_80/apache2", + vhost_root="debian_apache_2_4/two_vhost_80/apache2/sites-available"): + super(ParserTest, self).setUp(test_dir, config_root, vhost_root) zope.component.provideUtility(display_util.FileDisplay(sys.stdout)) @@ -55,11 +58,11 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): self.parser = ApacheParser( - self.aug, self.config_path, "dummy_ctl_path") + self.aug, self.config_path, self.vhost_path) def get_apache_configurator( - config_path, config_dir, work_dir, version=(2, 4, 7), conf=None): + config_path, vhost_path, config_dir, work_dir, version=(2, 4, 7), conf=None): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -68,7 +71,9 @@ def get_apache_configurator( backups = os.path.join(work_dir, "backups") mock_le_config = mock.MagicMock( apache_server_root=config_path, - apache_le_vhost_ext=constants.CLI_DEFAULTS["le_vhost_ext"], + apache_vhost_root=vhost_path, + apache_le_vhost_ext=constants.os_constant("le_vhost_ext"), + apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index a770804d1..2049eb574 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -1,14 +1,12 @@ """A class that performs TLS-SNI-01 challenges for Apache""" import os -import logging from letsencrypt.plugins import common from letsencrypt_apache import obj from letsencrypt_apache import parser -logger = logging.getLogger(__name__) class ApacheTlsSni01(common.TLSSNI01): """Class that performs TLS-SNI-01 challenges within the Apache configurator @@ -52,7 +50,7 @@ class ApacheTlsSni01(common.TLSSNI01): super(ApacheTlsSni01, self).__init__(*args, **kwargs) self.challenge_conf = os.path.join( - self.configurator.conf("server-root"), + self.configurator.conf("challenge-location"), "le_tls_sni_01_cert_challenge.conf") def perform(self): @@ -106,7 +104,6 @@ class ApacheTlsSni01(common.TLSSNI01): self.configurator.reverter.register_file_creation( True, self.challenge_conf) - logger.debug("writing a config file with text: %s", config_text) with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) diff --git a/letsencrypt-auto b/letsencrypt-auto index 44c71883c..20465dbb1 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -47,13 +47,13 @@ if test "`id -u`" -ne "0" ; then args="" # This `while` loop iterates over all parameters given to this function. # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrap in a pair of `'`, then append to `$args` string + # will be wrapped in a pair of `'`, then appended to `$args` string # For example, `echo "It's only 1\$\!"` will be escaped to: # 'echo' 'It'"'"'s only 1$!' # │ │└┼┘│ # │ │ │ └── `'s only 1$!'` the literal string # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings followed it + # │ └── `'It'`, to be concatenated with the strings following it # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself while [ $# -ne 0 ]; do args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " @@ -97,6 +97,7 @@ DeterminePythonVersion() { export LE_PYTHON=${LE_PYTHON:-python} else echo "Cannot find any Pythons... please install one!" + exit 1 fi PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` @@ -175,7 +176,7 @@ if [ "$VERBOSE" = 1 ] ; then echo $VENV_BIN/pip install -U setuptools $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -r "$LEA_PATH"/py26reqs.txt -U letsencrypt letsencrypt-apache + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache # nginx is buggy / disabled for now, but upgrade it if the user has # installed it manually if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then @@ -187,8 +188,6 @@ else $VENV_BIN/pip install -U pip > /dev/null printf . # nginx is buggy / disabled for now... - $VENV_BIN/pip install -r "$LEA_PATH"/py26reqs.txt > /dev/null - printf . $VENV_BIN/pip install -U letsencrypt > /dev/null printf . $VENV_BIN/pip install -U letsencrypt-apache > /dev/null @@ -201,5 +200,5 @@ fi # Explain what's about to happen, for the benefit of those getting sudo # password prompts... -echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" +echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" $SUDO $VENV_BIN/letsencrypt "$@" diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py index 5765003b9..ee679bdb7 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py @@ -15,11 +15,12 @@ from acme import crypto_util from acme import messages from letsencrypt import achallenges from letsencrypt import errors as le_errors -from letsencrypt import validator from letsencrypt.tests import acme_util from letsencrypt_compatibility_test import errors from letsencrypt_compatibility_test import util +from letsencrypt_compatibility_test import validator + from letsencrypt_compatibility_test.configurators.apache import apache24 diff --git a/letsencrypt/validator.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator.py similarity index 100% rename from letsencrypt/validator.py rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator.py diff --git a/letsencrypt/tests/validator_test.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator_test.py similarity index 77% rename from letsencrypt/tests/validator_test.py rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator_test.py index c7416dc46..3a3bbc4b2 100644 --- a/letsencrypt/tests/validator_test.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/validator_test.py @@ -1,4 +1,4 @@ -"""Tests for letsencrypt.validator.""" +"""Tests for letsencrypt_compatibility_test.validator.""" import requests import unittest @@ -6,28 +6,31 @@ import mock import OpenSSL from acme import errors as acme_errors -from letsencrypt import validator +from letsencrypt_compatibility_test import validator class ValidatorTest(unittest.TestCase): def setUp(self): self.validator = validator.Validator() - @mock.patch("letsencrypt.validator.crypto_util.probe_sni") + @mock.patch( + "letsencrypt_compatibility_test.validator.crypto_util.probe_sni") def test_certificate_success(self, mock_probe_sni): cert = OpenSSL.crypto.X509() mock_probe_sni.return_value = cert self.assertTrue(self.validator.certificate( cert, "test.com", "127.0.0.1")) - @mock.patch("letsencrypt.validator.crypto_util.probe_sni") + @mock.patch( + "letsencrypt_compatibility_test.validator.crypto_util.probe_sni") def test_certificate_error(self, mock_probe_sni): cert = OpenSSL.crypto.X509() mock_probe_sni.side_effect = [acme_errors.Error] self.assertFalse(self.validator.certificate( cert, "test.com", "127.0.0.1")) - @mock.patch("letsencrypt.validator.crypto_util.probe_sni") + @mock.patch( + "letsencrypt_compatibility_test.validator.crypto_util.probe_sni") def test_certificate_failure(self, mock_probe_sni): cert = OpenSSL.crypto.X509() cert.set_serial_number(1337) @@ -35,67 +38,67 @@ class ValidatorTest(unittest.TestCase): self.assertFalse(self.validator.certificate( cert, "test.com", "127.0.0.1")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_succesful_redirect(self, mock_get_request): mock_get_request.return_value = create_response( 301, {"location": "https://test.com"}) self.assertTrue(self.validator.redirect("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_redirect_with_headers(self, mock_get_request): mock_get_request.return_value = create_response( 301, {"location": "https://test.com"}) self.assertTrue(self.validator.redirect( "test.com", headers={"Host": "test.com"})) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_redirect_missing_location(self, mock_get_request): mock_get_request.return_value = create_response(301) self.assertFalse(self.validator.redirect("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_redirect_wrong_status_code(self, mock_get_request): mock_get_request.return_value = create_response( 201, {"location": "https://test.com"}) self.assertFalse(self.validator.redirect("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_redirect_wrong_redirect_code(self, mock_get_request): mock_get_request.return_value = create_response( 303, {"location": "https://test.com"}) self.assertFalse(self.validator.redirect("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_hsts_empty(self, mock_get_request): mock_get_request.return_value = create_response( headers={"strict-transport-security": ""}) self.assertFalse(self.validator.hsts("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_hsts_malformed(self, mock_get_request): mock_get_request.return_value = create_response( headers={"strict-transport-security": "sdfal"}) self.assertFalse(self.validator.hsts("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_hsts_bad_max_age(self, mock_get_request): mock_get_request.return_value = create_response( headers={"strict-transport-security": "max-age=not-an-int"}) self.assertFalse(self.validator.hsts("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_hsts_expire(self, mock_get_request): mock_get_request.return_value = create_response( headers={"strict-transport-security": "max-age=3600"}) self.assertFalse(self.validator.hsts("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_hsts(self, mock_get_request): mock_get_request.return_value = create_response( headers={"strict-transport-security": "max-age=31536000"}) self.assertTrue(self.validator.hsts("test.com")) - @mock.patch("letsencrypt.validator.requests.get") + @mock.patch("letsencrypt_compatibility_test.validator.requests.get") def test_hsts_include_subdomains(self, mock_get_request): mock_get_request.return_value = create_response( headers={"strict-transport-security": diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index eb7e23036..1ff9e7649 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -10,6 +10,7 @@ install_requires = [ 'letsencrypt=={0}'.format(version), 'letsencrypt-apache=={0}'.format(version), 'docker-py', + 'requests', 'zope.interface', ] @@ -18,6 +19,11 @@ if sys.version_info < (2, 7): else: install_requires.append('mock') +if sys.version_info < (2, 7, 9): + # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) + install_requires.append('ndg-httpsclient') + install_requires.append('pyasn1') + docs_extras = [ 'repoze.sphinx.autointerface', 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index aaaf43c5f..89b1145e7 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -311,17 +311,11 @@ class NginxConfigurator(common.Plugin): """ snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() ssl_block = [['listen', '{0} ssl'.format(self.config.tls_sni_01_port)], - # access and error logs necessary for integration - # testing (non-root) - ['access_log', os.path.join( - self.config.work_dir, 'access.log')], - ['error_log', os.path.join( - self.config.work_dir, 'error.log')], ['ssl_certificate', snakeoil_cert], ['ssl_certificate_key', snakeoil_key], ['include', self.parser.loc["ssl_options"]]] self.parser.add_server_directives( - vhost.filep, vhost.names, ssl_block) + vhost.filep, vhost.names, ssl_block, replace=False) vhost.ssl = True vhost.raw.extend(ssl_block) vhost.addrs.add(obj.Addr( @@ -384,7 +378,7 @@ class NginxConfigurator(common.Plugin): [['return', '301 https://$host$request_uri']] ]] self.parser.add_server_directives( - vhost.filep, vhost.names, redirect_block) + vhost.filep, vhost.names, redirect_block, replace=False) logger.info("Redirecting all traffic to ssl in %s", vhost.filep) ###################################### @@ -393,11 +387,10 @@ class NginxConfigurator(common.Plugin): def restart(self): """Restarts nginx server. - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If either the reload fails. """ - return nginx_restart(self.conf('ctl'), self.nginx_conf) + nginx_restart(self.conf('ctl'), self.nginx_conf) def config_test(self): # pylint: disable=no-self-use """Check the configuration of Nginx for errors. @@ -631,19 +624,16 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): if nginx_proc.returncode != 0: # Enter recovery routine... - logger.error("Nginx Restart Failed!\n%s\n%s", stdout, stderr) - return False + raise errors.MisconfigurationError( + "nginx restart failed:\n%s\n%s" % (stdout, stderr)) except (OSError, ValueError): - logger.fatal("Nginx Restart Failed - Please Check the Configuration") - sys.exit(1) + raise errors.MisconfigurationError("nginx restart failed") # Nginx can take a moment to recognize a newly added TLS SNI servername, so sleep # for a second. TODO: Check for expected servername and loop until it # appears or return an error if looping too long. time.sleep(1) - return True - def temp_install(options_ssl): """Temporary install for convenience.""" diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index 14db2f8b7..c60d0102a 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -213,6 +213,7 @@ class NginxParser(object): if ext: filename = filename + os.path.extsep + ext try: + logger.debug('Dumping to %s:\n%s', filename, nginxparser.dumps(tree)) with open(filename, 'w') as _file: nginxparser.dump(tree, _file) except IOError: @@ -252,7 +253,7 @@ class NginxParser(object): return server_names == names def add_server_directives(self, filename, names, directives, - replace=False): + replace): """Add or replace directives in the first server block with names. ..note :: If replace is True, this raises a misconfiguration error @@ -269,20 +270,27 @@ class NginxParser(object): :param bool replace: Whether to only replace existing directives """ - _do_for_subarray(self.parsed[filename], - lambda x: self._has_server_names(x, names), - lambda x: _add_directives(x, directives, replace)) + try: + _do_for_subarray(self.parsed[filename], + lambda x: self._has_server_names(x, names), + lambda x: _add_directives(x, directives, replace)) + except errors.MisconfigurationError as err: + raise errors.MisconfigurationError("Problem in %s: %s" % (filename, err.message)) def add_http_directives(self, filename, directives): """Adds directives to the first encountered HTTP block in filename. + We insert new directives at the top of the block to work around + https://trac.nginx.org/nginx/ticket/810: If the first server block + doesn't enable OCSP stapling, stapling is broken for all blocks. + :param str filename: The absolute filename of the config file :param list directives: The directives to add """ _do_for_subarray(self.parsed[filename], lambda x: x[0] == ['http'], - lambda x: _add_directives(x[1], [directives], False)) + lambda x: x[1].insert(0, directives)) def get_all_certs_keys(self): """Gets all certs and keys in the nginx config. @@ -467,9 +475,14 @@ def _parse_server(server): return parsed_server -def _add_directives(block, directives, replace=False): - """Adds or replaces directives in a block. If the directive doesn't exist in - the entry already, raises a misconfiguration error. +def _add_directives(block, directives, replace): + """Adds or replaces directives in a config block. + + When replace=False, it's an error to try and add a directive that already + exists in the config block with a conflicting value. + + When replace=True, a directive with the same name MUST already exist in the + config block, and the first instance will be replaced. ..todo :: Find directives that are in included files. @@ -478,21 +491,43 @@ def _add_directives(block, directives, replace=False): """ for directive in directives: - if not replace: - # We insert new directives at the top of the block, mostly - # to work around https://trac.nginx.org/nginx/ticket/810 - # Only add directive if its not already in the block - if directive not in block: - block.insert(0, directive) - else: - changed = False - if len(directive) == 0: - continue - for index, line in enumerate(block): - if len(line) > 0 and line[0] == directive[0]: - block[index] = directive - changed = True - if not changed: + _add_directive(block, directive, replace) + +repeatable_directives = set(['server_name', 'listen', 'include']) + +def _add_directive(block, directive, replace): + """Adds or replaces a single directive in a config block. + + See _add_directives for more documentation. + + """ + location = -1 + # Find the index of a config line where the name of the directive matches + # the name of the directive we want to add. + for index, line in enumerate(block): + if len(line) > 0 and line[0] == directive[0]: + location = index + break + if replace: + if location == -1: + raise errors.MisconfigurationError( + 'expected directive for %s in the Nginx ' + 'config but did not find it.' % directive[0]) + block[location] = directive + else: + # Append directive. Fail if the name is not a repeatable directive name, + # and there is already a copy of that directive with a different value + # in the config file. + directive_name = directive[0] + directive_value = directive[1] + if location != -1 and directive_name.__str__() not in repeatable_directives: + if block[location][1] == directive_value: + # There's a conflict, but the existing value matches the one we + # want to insert, so it's fine. + pass + else: raise errors.MisconfigurationError( - 'Let\'s Encrypt expected directive for %s in the Nginx ' - 'config but did not find it.' % directive[0]) + 'tried to insert directive "%s" but found conflicting "%s".' % ( + directive, block[location])) + else: + block.append(directive) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..171fd1f10 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -40,6 +40,23 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEquals((1, 6, 2), self.config.version) self.assertEquals(5, len(self.config.parser.parsed)) + @mock.patch("letsencrypt_nginx.configurator.le_util.exe_exists") + @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") + def test_prepare_initializes_version(self, mock_popen, mock_exe_exists): + mock_popen().communicate.return_value = ( + "", "\n".join(["nginx version: nginx/1.6.2", + "built by clang 6.0 (clang-600.0.56)" + " (based on LLVM 3.5svn)", + "TLS SNI support enabled", + "configure arguments: --prefix=/usr/local/Cellar/" + "nginx/1.6.2 --with-http_ssl_module"])) + + mock_exe_exists.return_value = True + + self.config.version = None + self.config.prepare() + self.assertEquals((1, 6, 2), self.config.version) + @mock.patch("letsencrypt_nginx.configurator.socket.gethostbyaddr") def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) @@ -65,16 +82,19 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') self.config.parser.add_server_directives( filep, set(['.example.com', 'example.*']), - [['listen', '5001 ssl']]) + [['listen', '5001 ssl']], + replace=False) self.config.save() # pylint: disable=protected-access parsed = self.config.parser._parse_files(filep, override=True) - self.assertEqual([[['server'], [['listen', '5001 ssl'], + self.assertEqual([[['server'], [ ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*']]]], + ['server_name', 'example.*'], + ['listen', '5001 ssl'] + ]]], parsed[0]) def test_choose_vhost(self): @@ -91,12 +111,26 @@ class NginxConfiguratorTest(util.NginxTest): 'test.www.example.com': foo_conf, 'abc.www.foo.com': foo_conf, 'www.bar.co.uk': localhost_conf} + + conf_path = {'localhost': "etc_nginx/nginx.conf", + 'alias': "etc_nginx/nginx.conf", + 'example.com': "etc_nginx/sites-enabled/example.com", + 'example.com.uk.test': "etc_nginx/sites-enabled/example.com", + 'www.example.com': "etc_nginx/sites-enabled/example.com", + 'test.www.example.com': "etc_nginx/foo.conf", + 'abc.www.foo.com': "etc_nginx/foo.conf", + 'www.bar.co.uk': "etc_nginx/nginx.conf"} + bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] for name in results: - self.assertEqual(results[name], - self.config.choose_vhost(name).names) + vhost = self.config.choose_vhost(name) + path = os.path.relpath(vhost.filep, self.temp_dir) + + self.assertEqual(results[name], vhost.names) + self.assertEqual(conf_path[name], path) + for name in bad_results: self.assertEqual(set([name]), self.config.choose_vhost(name).names) @@ -154,38 +188,36 @@ class NginxConfiguratorTest(util.NginxTest): parsed_server_conf = util.filter_comments(self.config.parser.parsed[server_conf]) parsed_nginx_conf = util.filter_comments(self.config.parser.parsed[nginx_conf]) - access_log = os.path.join(self.work_dir, "access.log") - error_log = os.path.join(self.work_dir, "error.log") self.assertEqual([[['server'], - [['include', self.config.parser.loc["ssl_options"]], - ['ssl_certificate_key', 'example/key.pem'], - ['ssl_certificate', 'example/fullchain.pem'], - ['error_log', error_log], - ['access_log', access_log], - - ['listen', '5001 ssl'], + [ ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*']]]], + ['server_name', 'example.*'], + + ['listen', '5001 ssl'], + ['ssl_certificate', 'example/fullchain.pem'], + ['ssl_certificate_key', 'example/key.pem'], + ['include', self.config.parser.loc["ssl_options"]] + ]]], parsed_example_conf) self.assertEqual([['server_name', 'somename alias another.alias']], parsed_server_conf) - self.assertTrue(util.contains_at_depth(parsed_nginx_conf, - [['server'], - [['include', self.config.parser.loc["ssl_options"]], - ['ssl_certificate_key', '/etc/nginx/key.pem'], - ['ssl_certificate', '/etc/nginx/fullchain.pem'], - ['error_log', error_log], - ['access_log', access_log], - ['listen', '5001 ssl'], - ['listen', '8000'], - ['listen', 'somename:8080'], - ['include', 'server.conf'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html index.htm']]]]], - 2)) + self.assertTrue(util.contains_at_depth( + parsed_nginx_conf, + [['server'], + [ + ['listen', '8000'], + ['listen', 'somename:8080'], + ['include', 'server.conf'], + [['location', '/'], + [['root', 'html'], + ['index', 'index.html index.htm']]], + ['listen', '5001 ssl'], + ['ssl_certificate', '/etc/nginx/fullchain.pem'], + ['ssl_certificate_key', '/etc/nginx/key.pem'], + ['include', self.config.parser.loc["ssl_options"]]]], + 2)) def test_get_all_certs_keys(self): nginx_conf = self.config.parser.abs_path('nginx.conf') @@ -297,19 +329,19 @@ class NginxConfiguratorTest(util.NginxTest): mocked = mock_popen() mocked.communicate.return_value = ('', '') mocked.returncode = 0 - self.assertTrue(self.config.restart()) + self.config.restart() @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_nginx_restart_fail(self, mock_popen): mocked = mock_popen() mocked.communicate.return_value = ('', '') mocked.returncode = 1 - self.assertFalse(self.config.restart()) + self.assertRaises(errors.MisconfigurationError, self.config.restart) @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_no_nginx_start(self, mock_popen): mock_popen.side_effect = OSError("Can't find program") - self.assertRaises(SystemExit, self.config.restart) + self.assertRaises(errors.MisconfigurationError, self.config.restart) @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_config_test(self, mock_popen): @@ -330,6 +362,17 @@ class NginxConfiguratorTest(util.NginxTest): OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, key_file.read()) + def test_redirect_enhance(self): + expected = [ + ['if', '($scheme != "https")'], + [['return', '301 https://$host$request_uri']] + ] + + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.enhance("www.example.com", "redirect") + + generated_conf = self.config.parser.parsed[example_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 2d6156429..6559a5df6 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -127,7 +127,8 @@ class NginxParserTest(util.NginxTest): set(['localhost', r'~^(www\.)?(example|bar)\.']), [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert.pem']]) + '/etc/ssl/cert.pem']], + replace=False) ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')]) self.assertEqual(1, len(re.findall(ssl_re, dump))) @@ -136,12 +137,15 @@ class NginxParserTest(util.NginxTest): names = set(['alias', 'another.alias', 'somename']) nparser.add_server_directives(server_conf, names, [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert2.pem']]) - nparser.add_server_directives(server_conf, names, [['foo', 'bar']]) + '/etc/ssl/cert2.pem']], + replace=False) + nparser.add_server_directives(server_conf, names, [['foo', 'bar']], + replace=False) self.assertEqual(nparser.parsed[server_conf], - [['ssl_certificate', '/etc/ssl/cert2.pem'], + [['server_name', 'somename alias another.alias'], ['foo', 'bar'], - ['server_name', 'somename alias another.alias']]) + ['ssl_certificate', '/etc/ssl/cert2.pem'] + ]) def test_add_http_directives(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) @@ -165,17 +169,19 @@ class NginxParserTest(util.NginxTest): target = set(['.example.com', 'example.*']) filep = nparser.abs_path('sites-enabled/example.com') nparser.add_server_directives( - filep, target, [['server_name', 'foo bar']], True) + filep, target, [['server_name', 'foobar.com']], replace=True) self.assertEqual( nparser.parsed[filep], [[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], - ['server_name', 'foo bar'], - ['server_name', 'foo bar']]]]) + ['server_name', 'foobar.com'], + ['server_name', 'example.*'], + ]]]) self.assertRaises(errors.MisconfigurationError, nparser.add_server_directives, - filep, set(['foo', 'bar']), - [['ssl_certificate', 'cert.pem']], True) + filep, set(['foobar.com', 'example.*']), + [['ssl_certificate', 'cert.pem']], + replace=True) def test_get_best_match(self): target_name = 'www.eff.org' @@ -217,7 +223,8 @@ class NginxParserTest(util.NginxTest): set(['.example.com', 'example.*']), [['ssl_certificate', 'foo.pem'], ['ssl_certificate_key', 'bar.key'], - ['listen', '443 ssl']]) + ['listen', '443 ssl']], + replace=False) c_k = nparser.get_all_certs_keys() self.assertEqual(set([('foo.pem', 'bar.key', filep)]), c_k) diff --git a/letsencrypt-nginx/tests/boulder-integration.conf.sh b/letsencrypt-nginx/tests/boulder-integration.conf.sh index 12610d895..d77669a76 100755 --- a/letsencrypt-nginx/tests/boulder-integration.conf.sh +++ b/letsencrypt-nginx/tests/boulder-integration.conf.sh @@ -20,13 +20,14 @@ events { } http { - # Set an array of temp and cache file options that will otherwise default to + # Set an array of temp, cache and log file options that will otherwise default to # restricted locations accessible only to root. client_body_temp_path $root/client_body; fastcgi_temp_path $root/fastcgi_temp; proxy_temp_path $root/proxy_temp; #scgi_temp_path $root/scgi_temp; #uwsgi_temp_path $root/uwsgi_temp; + access_log $root/error.log; # This should be turned off in a Virtualbox VM, as it can cause some # interesting issues with data corruption in delivered files. @@ -54,9 +55,6 @@ http { root $root/webroot; - access_log $root/access.log; - error_log $root/error.log; - location / { # First attempt to serve request as file, then as directory, then fall # back to index.html. diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 027c11158..c63d8c8d4 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -540,16 +540,11 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ - msg = [ - "The following '{0}' errors were reported by the server:".format(typ)] + msg = ["The following errors were reported by the server:"] - problems = dict() for achall in failed_achalls: - problems.setdefault(achall.error.description, set()).add(achall.domain) - for problem in problems: - msg.append("\n\nDomains: ") - msg.append(", ".join(sorted(problems[problem]))) - msg.append("\nError: {0}".format(problem)) + msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % ( + achall.domain, achall.error.typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 29519d430..aba9116f9 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1189,7 +1189,7 @@ def _plugins_parsing(helpful, plugins): # These would normally be a flag within the webroot plugin, but because # they are parsed in conjunction with --domains, they live here for - # legibiility. helpful.add_plugin_ags must be called first to add the + # legibility. helpful.add_plugin_ags must be called first to add the # "webroot" topic helpful.add("webroot", "-w", "--webroot-path", action=WebrootPathProcessor, help="public_html / webroot path. This can be specified multiple times to " diff --git a/letsencrypt/plugins/webroot_test.py b/letsencrypt/plugins/webroot_test.py index 9f5b6bba8..defe9396b 100644 --- a/letsencrypt/plugins/webroot_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -66,8 +66,16 @@ class AuthenticatorTest(unittest.TestCase): def test_prepare_reraises_other_errors(self): self.auth.full_path = os.path.join(self.path, "null") + permission_canary = os.path.join(self.path, "rnd") + with open(permission_canary, "w") as f: + f.write("thingimy") os.chmod(self.path, 0o000) - self.assertRaises(errors.PluginError, self.auth.prepare) + try: + open(permission_canary, "r") + print "Warning, running tests as root skips permissions tests..." + except IOError: + # ok, permissions work, test away... + self.assertRaises(errors.PluginError, self.auth.prepare) os.chmod(self.path, 0o700) @mock.patch("letsencrypt.plugins.webroot.os.chown") diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 3996cfe67..2f36e7e91 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -116,6 +116,7 @@ def renew(cert, old_version): def _cli_log_handler(args, level, fmt): # pylint: disable=unused-argument handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) + handler.setLevel(level) return handler @@ -181,7 +182,9 @@ def main(cli_args=sys.argv[1:]): # RenewableCert object for this cert at all, which could # dramatically improve performance for large deployments # where autorenewal is widely turned off. - cert = storage.RenewableCert(renewal_file, cli_config) + cert = storage.RenewableCert( + os.path.join(cli_config.renewal_configs_dir, renewal_file), + cli_config) except errors.CertStorageError: # This indicates an invalid renewal configuration file, such # as one missing a required parameter (in the future, perhaps diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 3b2b548b0..ac71bd9fe 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -260,7 +260,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :returns: The path to the current version of the specified member. - :rtype: str + :rtype: str or None """ if kind not in ALL_FOUR: @@ -450,12 +450,15 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :param int version: the desired version number :returns: the subject names :rtype: `list` of `str` + :raises .CertStorageError: if could not find cert file. """ if version is None: target = self.current_target("cert") else: target = self.version("cert", version) + if target is None: + raise errors.CertStorageError("could not find cert file") with open(target) as f: return crypto_util.get_sans_from_cert(f.read()) diff --git a/letsencrypt/tests/auth_handler_test.py b/letsencrypt/tests/auth_handler_test.py index be19ab036..5b4c2bfc7 100644 --- a/letsencrypt/tests/auth_handler_test.py +++ b/letsencrypt/tests/auth_handler_test.py @@ -467,7 +467,7 @@ class ReportFailedChallsTest(unittest.TestCase): auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) - self.assertTrue("Domains: example.com\n" in call_list[0][0][0]) + self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @mock.patch("letsencrypt.auth_handler.zope.component.getUtility") def test_different_errors_and_domains(self, mock_zope): diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index daec9678f..d583e8645 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -764,6 +764,8 @@ class RenewableCertTests(BaseRenewableCertTest): def test_bad_config_file(self): from letsencrypt import renewer + os.unlink(os.path.join(self.cli_config.renewal_configs_dir, + "example.org.conf")) with open(os.path.join(self.cli_config.renewal_configs_dir, "bad.conf"), "w") as f: f.write("incomplete = configfile\n") diff --git a/py26reqs.txt b/py26reqs.txt deleted file mode 100644 index a94b22c0c..000000000 --- a/py26reqs.txt +++ /dev/null @@ -1,2 +0,0 @@ -# https://github.com/bw2/ConfigArgParse/issues/17 -git+https://github.com/kuba/ConfigArgParse.git@python2.6-0.9.3#egg=ConfigArgParse diff --git a/setup.py b/setup.py index 40c6ac16c..f95f672ff 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ version = meta['version'] install_requires = [ 'acme=={0}'.format(version), - 'ConfigArgParse', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', @@ -41,7 +40,6 @@ install_requires = [ 'pyrfc3339', 'python2-pythondialog>=3.2.2rc1', # Debian squeeze support, cf. #280 'pytz', - 'requests', 'setuptools', # pkg_resources 'six', 'zope.component', @@ -53,10 +51,14 @@ if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', + 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 'mock<1.1.0', ]) else: - install_requires.append('mock') + install_requires.extend([ + 'ConfigArgParse', + 'mock', + ]) dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test deleted file mode 100755 index c6663551e..000000000 --- a/tests/apache-conf-files/hackish-apache-test +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# A hackish script to see if the client is behaving as expected -# with each of the "passing" conf files. - -# TODO presently this requires interaction and human judgement to -# assess, but it should be automated -export EA=/etc/apache2/ -TESTDIR="`dirname $0`" -LEROOT="`realpath \"$TESTDIR/../../\"`" -cd $TESTDIR/passing - -function CleanupExit() { - echo control c, exiting tests... - if [ "$f" != "" ] ; then - sudo rm /etc/apache2/sites-{enabled,available}/"$f" - fi - exit 1 -} - -trap CleanupExit INT -for f in *.conf ; do - echo testing "$f" - sudo cp "$f" "$EA"/sites-available/ - sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - sudo "$LEROOT"/venv/bin/letsencrypt --apache certonly -t - sudo rm /etc/apache2/sites-{enabled,available}/"$f" -done diff --git a/tests/apache-conf-files/passing/README.modules b/tests/apache-conf-files/passing/README.modules deleted file mode 100644 index 7edbd3e84..000000000 --- a/tests/apache-conf-files/passing/README.modules +++ /dev/null @@ -1,7 +0,0 @@ -Modules required to parse these conf files: - -ssl -rewrite -macro -wsgi -deflate diff --git a/tests/letstest/README.md b/tests/letstest/README.md new file mode 100644 index 000000000..a085e9d91 --- /dev/null +++ b/tests/letstest/README.md @@ -0,0 +1,44 @@ +# letstest +simple aws testfarm scripts for letsencrypt client testing + +- Configures (canned) boulder server +- Launches EC2 instances with a given list of AMIs for different distros +- Copies letsencrypt repo and puts it on the instances +- Runs letsencrypt tests (bash scripts) on all of these +- Logs execution and success/fail for debugging + +## Notes + - Some AWS images, e.g. official CentOS and FreeBSD images + require acceptance of user terms on the AWS marketplace + website. This can't be automated. + - AWS EC2 has a default limit of 20 t2/t1 instances, if more + are needed, they need to be requested via online webform. + +## Usage + - Requires AWS IAM secrets to be set up with aws cli + - Requires an AWS associated keyfile .pem + +``` +>aws configure --profile HappyHacker +[interactive: enter secrets for IAM role] +>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair --query 'KeyMaterial' --output text > MyKeyPair.pem +``` +then: +``` +>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_apache2.sh +``` + +## Scripts +example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed +to them at runtime via environment variables. test_apache2.sh is a useful reference. + +Note that the 
test_letsencrypt_auto_*
scripts pull code from PyPI using the letsencrypt-auto script, +__not__ the local python code. test_apache2 runs the dev venv and does local tests. + +see: +- https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html +- https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html + +main repos: +- https://github.com/letsencrypt/boulder +- https://github.com/letsencrypt/letsencrypt diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml new file mode 100644 index 000000000..e707b8636 --- /dev/null +++ b/tests/letstest/apache2_targets.yaml @@ -0,0 +1,57 @@ +targets: + #----------------------------------------------------------------------------- + # Apache 2.4 + - ami: ami-26d5af4c + name: ubuntu15.10 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-d92e6bb3 + name: ubuntu15.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-7b89cc11 + name: ubuntu14.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-9295d0f8 + name: ubuntu14.04LTS_32bit + type: ubuntu + virt: pv + user: ubuntu + - ami: ami-116d857a + name: debian8.1 + type: debian + virt: hvm + user: admin + userdata: | + #cloud-init + runcmd: + - [ apt-get, install, -y, curl ] + #----------------------------------------------------------------------------- + # Apache 2.2 + # - ami: ami-0611546c + # name: ubuntu12.04LTS + # type: ubuntu + # virt: hvm + # user: ubuntu + # - ami: ami-e0efab88 + # name: debian7.8.aws.1 + # type: debian + # virt: hvm + # user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + # - ami: ami-e6eeaa8e + # name: debian7.8.aws.1_32bit + # type: debian + # virt: pv + # user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] \ No newline at end of file diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py new file mode 100644 index 000000000..dee6968c3 --- /dev/null +++ b/tests/letstest/multitester.py @@ -0,0 +1,528 @@ +""" +Letsencrypt Integration Test Tool + +- Configures (canned) boulder server +- Launches EC2 instances with a given list of AMIs for different distros +- Copies letsencrypt repo and puts it on the instances +- Runs letsencrypt tests (bash scripts) on all of these +- Logs execution and success/fail for debugging + +Notes: + - Some AWS images, e.g. official CentOS and FreeBSD images + require acceptance of user terms on the AWS marketplace + website. This can't be automated. + - AWS EC2 has a default limit of 20 t2/t1 instances, if more + are needed, they need to be requested via online webform. + +Usage: + - Requires AWS IAM secrets to be set up with aws cli + - Requires an AWS associated keyfile .pem + +>aws configure --profile HappyHacker +[interactive: enter secrets for IAM role] +>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \ + --query 'KeyMaterial' --output text > MyKeyPair.pem +then: +>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh +see: + https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html + https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html +""" + +from __future__ import print_function +from __future__ import with_statement + +import sys, os, time, argparse, socket +import multiprocessing as mp +from multiprocessing import Manager +import urllib2 +import yaml +import boto3 +import fabric +from fabric.api import run, execute, local, env, sudo, cd, lcd +from fabric.operations import get, put +from fabric.context_managers import shell_env + +# Command line parser +#------------------------------------------------------------------------------- +parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.') +parser.add_argument('config_file', + help='yaml configuration file for AWS server cluster') +parser.add_argument('key_file', + help='key file (.pem) for AWS') +parser.add_argument('aws_profile', + help='profile for AWS (i.e. as in ~/.aws/certificates)') +parser.add_argument('test_script', + default='test_letsencrypt_auto_certonly_standalone.sh', + help='path of bash script in to deploy and run') +#parser.add_argument('--script_args', +# nargs='+', +# help='space-delimited list of arguments to pass to the bash test script', +# required=False) +parser.add_argument('--repo', + default='https://github.com/letsencrypt/letsencrypt.git', + help='letsencrypt git repo to use') +parser.add_argument('--branch', + default='~', + help='letsencrypt git branch to trial') +parser.add_argument('--pull_request', + default='~', + help='letsencrypt/letsencrypt pull request to trial') +parser.add_argument('--merge_master', + action='store_true', + help="if set merges PR into master branch of letsencrypt/letsencrypt") +parser.add_argument('--saveinstances', + action='store_true', + help="don't kill EC2 instances after run, useful for debugging") +parser.add_argument('--alt_pip', + default='', + help="server from which to pull candidate release packages") +parser.add_argument('--killboulder', + action='store_true', + help="do not leave a persistent boulder server running") +parser.add_argument('--boulderonly', + action='store_true', + help="only make a boulder server") +parser.add_argument('--fast', + action='store_true', + help="use larger instance types to run faster (saves about a minute, probably not worth it)") +cl_args = parser.parse_args() + +# Credential Variables +#------------------------------------------------------------------------------- +# assumes naming: = .pem +KEYFILE = cl_args.key_file +KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0] +PROFILE = cl_args.aws_profile + +# Globals +#------------------------------------------------------------------------------- +BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1 +LOGDIR = "" #points to logging / working directory +# boto3/AWS api globals +AWS_SESSION = None +EC2 = None + +# Boto3/AWS automation functions +#------------------------------------------------------------------------------- +def make_security_group(): + # will fail if security group of GroupName already exists + # cannot have duplicate SGs of the same name + mysg = EC2.create_security_group(GroupName="letsencrypt_test", + Description='security group for automated testing') + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443) + # for boulder wfe (http) server + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000) + # for mosh + mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000) + return mysg + +def make_instance(instance_name, + ami_id, + keyname, + machine_type='t2.micro', + security_groups=['letsencrypt_test'], + userdata=""): #userdata contains bash or cloud-init script + + new_instance = EC2.create_instances( + ImageId=ami_id, + SecurityGroups=security_groups, + KeyName=keyname, + MinCount=1, + MaxCount=1, + UserData=userdata, + InstanceType=machine_type)[0] + + # brief pause to prevent rare error on EC2 delay, should block until ready instead + time.sleep(1.0) + + # give instance a name + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + return new_instance + +def terminate_and_clean(instances): + """ + Some AMIs specify EBS stores that won't delete on instance termination. + These must be manually deleted after shutdown. + """ + volumes_to_delete = [] + for instance in instances: + for bdmap in instance.block_device_mappings: + if 'Ebs' in bdmap.keys(): + if not bdmap['Ebs']['DeleteOnTermination']: + volumes_to_delete.append(bdmap['Ebs']['VolumeId']) + + for instance in instances: + instance.terminate() + + # can't delete volumes until all attaching instances are terminated + _ids = [instance.id for instance in instances] + all_terminated = False + while not all_terminated: + all_terminated = True + for _id in _ids: + # necessary to reinit object for boto3 to get true state + inst = EC2.Instance(id=_id) + if inst.state['Name'] != 'terminated': + all_terminated = False + time.sleep(5) + + for vol_id in volumes_to_delete: + volume = EC2.Volume(id=vol_id) + volume.delete() + + return volumes_to_delete + + +# Helper Routines +#------------------------------------------------------------------------------- +def block_until_http_ready(urlstring, wait_time=10, timeout=240): + "Blocks until server at urlstring can respond to http requests" + server_ready = False + t_elapsed = 0 + while not server_ready and t_elapsed < timeout: + try: + sys.stdout.write('.') + sys.stdout.flush() + req = urllib2.Request(urlstring) + response = urllib2.urlopen(req) + #if response.code == 200: + server_ready = True + except urllib2.URLError: + pass + time.sleep(wait_time) + t_elapsed += wait_time + +def block_until_ssh_open(ipstring, wait_time=10, timeout=120): + "Blocks until server at ipstring has an open port 22" + reached = False + t_elapsed = 0 + while not reached and t_elapsed < timeout: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((ipstring, 22)) + reached = True + except socket.error as err: + time.sleep(wait_time) + t_elapsed += wait_time + sock.close() + +def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20): + "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections" + # the reinstantiation from id is necessary to force boto3 + # to correctly update the 'state' variable during init + _id = booting_instance.id + _instance = EC2.Instance(id=_id) + _state = _instance.state['Name'] + _ip = _instance.public_ip_address + while _state != 'running' or _ip is None: + time.sleep(wait_time) + _instance = EC2.Instance(id=_id) + _state = _instance.state['Name'] + _ip = _instance.public_ip_address + block_until_ssh_open(_ip) + time.sleep(extra_wait_time) + return _instance + + +# Fabric Routines +#------------------------------------------------------------------------------- +def local_git_clone(repo_url): + "clones master of repo_url" + with lcd(LOGDIR): + local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') + local('git clone %s'% repo_url) + local('tar czf le.tar.gz letsencrypt') + +def local_git_branch(repo_url, branch_name): + "clones branch of repo_url" + with lcd(LOGDIR): + local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') + local('git clone %s --branch %s --single-branch'%(repo_url, branch_name)) + local('tar czf le.tar.gz letsencrypt') + +def local_git_PR(repo_url, PRnumstr, merge_master=True): + "clones specified pull request from repo_url and optionally merges into master" + with lcd(LOGDIR): + local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') + local('git clone %s'% repo_url) + local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) + local('cd letsencrypt && git co lePRtest') + if merge_master: + local('cd letsencrypt && git remote update origin') + local('cd letsencrypt && git merge origin/master -m "testmerge"') + local('tar czf le.tar.gz letsencrypt') + +def local_repo_to_remote(): + "copies local tarball of repo to remote" + with lcd(LOGDIR): + put(local_path='le.tar.gz', remote_path='') + run('tar xzf le.tar.gz') + +def local_repo_clean(): + "delete tarball" + with lcd(LOGDIR): + local('rm le.tar.gz') + +def deploy_script(scriptpath, *args): + "copies to remote and executes local script" + #with lcd('scripts'): + put(local_path=scriptpath, remote_path='', mirror_local_mode=True) + scriptfile = os.path.split(scriptpath)[1] + args_str = ' '.join(args) + run('./'+scriptfile+' '+args_str) + +def run_boulder(): + with cd('$GOPATH/src/github.com/letsencrypt/boulder'): + run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost') + run('nohup ./start.py >& /dev/null < /dev/null &') + +def config_and_launch_boulder(instance): + execute(deploy_script, 'scripts/boulder_config.sh') + execute(run_boulder) + +def install_and_launch_letsencrypt(instance, boulder_url, target): + execute(local_repo_to_remote) + with shell_env(BOULDER_URL=boulder_url, + PUBLIC_IP=instance.public_ip_address, + PRIVATE_IP=instance.private_ip_address, + PUBLIC_HOSTNAME=instance.public_dns_name, + PIP_EXTRA_INDEX_URL=cl_args.alt_pip, + OS_TYPE=target['type']): + execute(deploy_script, cl_args.test_script) + +def grab_letsencrypt_log(): + "grabs letsencrypt.log via cat into logged stdout" + sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \ + cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi') + # fallback file if /var/log is unwriteable...? correct? + sudo('if [ -f ./letsencrypt.log ]; then \ + cat ./letsencrypt.log; else echo "[nolocallog]"; fi') + +def create_client_instances(targetlist): + "Create a fleet of client instances" + instances = [] + print("Creating instances: ", end="") + for target in targetlist: + if target['virt'] == 'hvm': + machine_type = 't2.medium' if cl_args.fast else 't2.micro' + else: + # 32 bit systems + machine_type = 'c1.medium' if cl_args.fast else 't1.micro' + if 'userdata' in target.keys(): + userdata = target['userdata'] + else: + userdata = '' + name = 'le-%s'%target['name'] + print(name, end=" ") + instances.append(make_instance(name, + target['ami'], + KEYNAME, + machine_type=machine_type, + userdata=userdata)) + print() + return instances + +#------------------------------------------------------------------------------- +# SCRIPT BEGINS +#------------------------------------------------------------------------------- + +# Fabric library controlled through global env parameters +env.key_filename = KEYFILE +env.shell = '/bin/bash -l -i -c' +env.connection_attempts = 5 +env.timeout = 10 +# replace default SystemExit thrown by fabric during trouble +class FabricException(Exception): + pass +env['abort_exception'] = FabricException + +# Set up local copy of git repo +#------------------------------------------------------------------------------- +LOGDIR = "letest-%d"%int(time.time()) +print("Making local dir for test repo and logs: %s"%LOGDIR) +local('mkdir %s'%LOGDIR) + +# figure out what git object to test and locally create it in LOGDIR +print("Making local git repo") +try: + if cl_args.pull_request != '~': + print('Testing PR %s '%cl_args.pull_request, + "MERGING into master" if cl_args.merge_master else "") + execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master) + elif cl_args.branch != '~': + print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo)) + execute(local_git_branch, cl_args.repo, cl_args.branch) + else: + print('Testing master of %s'%cl_args.repo) + execute(local_git_clone, cl_args.repo) +except FabricException: + print("FAIL: trouble with git repo") + exit() + + +# Set up EC2 instances +#------------------------------------------------------------------------------- +configdata = yaml.load(open(cl_args.config_file, 'r')) +targetlist = configdata['targets'] +print('Testing against these images: [%d total]'%len(targetlist)) +for target in targetlist: + print(target['ami'], target['name']) + +print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE)) +AWS_SESSION = boto3.session.Session(profile_name=PROFILE) +EC2 = AWS_SESSION.resource('ec2') + +print("Making Security Group") +sg_exists = False +for sg in EC2.security_groups.all(): + if sg.group_name == 'letsencrypt_test': + sg_exists = True + print(" %s already exists"%'letsencrypt_test') +if not sg_exists: + make_security_group() + time.sleep(30) + +boulder_preexists = False +boulder_servers = EC2.instances.filter(Filters=[ + {'Name': 'tag:Name', 'Values': ['le-boulderserver']}, + {'Name': 'instance-state-name', 'Values': ['running']}]) + +boulder_server = next(iter(boulder_servers), None) + +print("Requesting Instances...") +if boulder_server: + print("Found existing boulder server:", boulder_server) + boulder_preexists = True +else: + print("Can't find a boulder server, starting one...") + boulder_server = make_instance('le-boulderserver', + BOULDER_AMI, + KEYNAME, + machine_type='t2.micro', + #machine_type='t2.medium', + security_groups=['letsencrypt_test']) + +if not cl_args.boulderonly: + instances = create_client_instances(targetlist) + +# Configure and launch boulder server +#------------------------------------------------------------------------------- +print("Waiting on Boulder Server") +boulder_server = block_until_instance_ready(boulder_server) +print(" server %s"%boulder_server) + + +# env.host_string defines the ssh user and host for connection +env.host_string = "ubuntu@%s"%boulder_server.public_ip_address +print("Boulder Server at (SSH):", env.host_string) +if not boulder_preexists: + print("Configuring and Launching Boulder") + config_and_launch_boulder(boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) + +boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address +print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) +print("Boulder Server at (EC2 private ip): %s"%boulder_url) + +if cl_args.boulderonly: + sys.exit(0) + +# Install and launch client scripts in parallel +#------------------------------------------------------------------------------- +print("Uploading and running test script in parallel: %s"%cl_args.test_script) +print("Output routed to log files in %s"%LOGDIR) +# (Advice: always use Manager.Queue, never regular multiprocessing.Queue +# the latter has implementation flaws that deadlock it in some circumstances) +manager = Manager() +outqueue = manager.Queue() +inqueue = manager.Queue() +SENTINEL = None #queue kill signal + +# launch as many processes as clients to test +num_processes = len(targetlist) +jobs = [] #keep a reference to current procs + +def test_client_process(inqueue, outqueue): + cur_proc = mp.current_process() + for inreq in iter(inqueue.get, SENTINEL): + ii, target = inreq + + #save all stdout to log file + sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w') + + print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) + instances[ii] = block_until_instance_ready(instances[ii]) + print("server %s at %s"%(instances[ii], instances[ii].public_ip_address)) + env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address) + print(env.host_string) + + try: + install_and_launch_letsencrypt(instances[ii], boulder_url, target) + outqueue.put((ii, target, 'pass')) + print("%s - %s SUCCESS"%(target['ami'], target['name'])) + except: + outqueue.put((ii, target, 'fail')) + print("%s - %s FAIL"%(target['ami'], target['name'])) + pass + + # append server letsencrypt.log to each per-machine output log + print("\n\nletsencrypt.log\n" + "-"*80 + "\n") + try: + execute(grab_letsencrypt_log) + except: + print("log fail\n") + pass + +# initiate process execution +for i in range(num_processes): + p = mp.Process(target=test_client_process, args=(inqueue, outqueue)) + jobs.append(p) + p.daemon = True # kills subprocesses if parent is killed + p.start() + +# fill up work queue +for ii, target in enumerate(targetlist): + inqueue.put((ii, target)) + +# add SENTINELs to end client processes +for i in range(num_processes): + inqueue.put(SENTINEL) +# wait on termination of client processes +for p in jobs: + p.join() +# add SENTINEL to output queue +outqueue.put(SENTINEL) + +# clean up +execute(local_repo_clean) + +# print and save summary results +results_file = open(LOGDIR+'/results', 'w') +outputs = [outq for outq in iter(outqueue.get, SENTINEL)] +outputs.sort(key=lambda x: x[0]) +for outq in outputs: + ii, target, status = outq + print('%d %s %s'%(ii, target['name'], status)) + results_file.write('%d %s %s\n'%(ii, target['name'], status)) +results_file.close() + +if not cl_args.saveinstances: + print('Logs in ', LOGDIR) + print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') + if cl_args.killboulder: + boulder_server.terminate() + terminate_and_clean(instances) +else: + # print login information for the boxes for debugging + for ii, target in enumerate(targetlist): + print(target['name'], + target['ami'], + "%s@%s"%(target['user'], instances[ii].public_ip_address)) + +# kill any connections +fabric.network.disconnect_all() diff --git a/tests/letstest/scripts/boulder_config.sh b/tests/letstest/scripts/boulder_config.sh new file mode 100755 index 000000000..1ef63ca10 --- /dev/null +++ b/tests/letstest/scripts/boulder_config.sh @@ -0,0 +1,32 @@ +#!/bin/bash -x + +# Configures and Launches Boulder Server installed on +# us-east-1 ami-5f490b35 bouldertestserver (boulder commit 8b433f54dab) + +# fetch instance data from EC2 metadata service +public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) +public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) +private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) + +# get local DNS resolver for VPC +resolver_ip=$(grep nameserver /etc/resolv.conf |cut -d" " -f2 |head -1) +resolver=$resolver_ip':53' + +# modifies integration testing boulder setup for local AWS VPC network +# connections instead of localhost +cd $GOPATH/src/github.com/letsencrypt/boulder +# configure boulder to receive outside connection on 4000 +sed -i '/listenAddress/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json +sed -i '/baseURL/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json +# change test ports to real +sed -i '/httpPort/ s/5002/80/' ./test/boulder-config.json +sed -i '/httpsPort/ s/5001/443/' ./test/boulder-config.json +sed -i '/tlsPort/ s/5001/443/' ./test/boulder-config.json +# set local dns resolver +sed -i '/dnsResolver/ s/127.0.0.1:8053/'$resolver'/' ./test/boulder-config.json + +# start rabbitMQ +#go run cmd/rabbitmq-setup/main.go -server amqp://localhost +# start acme services +#nohup ./start.py >& /dev/null < /dev/null & +#./start.py diff --git a/tests/letstest/scripts/boulder_install.sh b/tests/letstest/scripts/boulder_install.sh new file mode 100755 index 000000000..b5ddf2c5b --- /dev/null +++ b/tests/letstest/scripts/boulder_install.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x + +# >>>> only tested on Ubuntu 14.04LTS <<<< + +# non-interactive install of mariadb and other dependencies +export DEBIAN_FRONTEND=noninteractive +sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS' +sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS' +apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server +sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');" + +# install go +wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz +tar xzvf go1.5.1.linux-amd64.tar.gz +mkdir gocode +echo "export GOROOT=/home/ubuntu/go \n\ + export GOPATH=/home/ubuntu/gocode\n\ + export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc + +# install boulder and its go dependencies +go get -d github.com/letsencrypt/boulder/... +cd $GOPATH/src/github.com/letsencrypt/boulder +wget https://github.com/jsha/boulder-tools/raw/master/goose.gz +mkdir $GOPATH/bin +zcat goose.gz > $GOPATH/bin/goose +chmod +x $GOPATH/bin/goose +./test/create_db.sh +go get github.com/jsha/listenbuddy diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh new file mode 100755 index 000000000..4032e2195 --- /dev/null +++ b/tests/letstest/scripts/test_apache2.sh @@ -0,0 +1,74 @@ +#!/bin/bash -x + +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution + +if [ "$OS_TYPE" = "ubuntu" ] +then + CONFFILE=/etc/apache2/sites-available/000-default.conf + sudo apt-get update + sudo apt-get -y --no-upgrade install apache2 #curl + sudo apt-get -y install realpath # needed for test-apache-conf + # For apache 2.4, set up ServerName + sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE + sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE +elif [ "$OS_TYPE" = "centos" ] +then + CONFFILE=/etc/httpd/conf/httpd.conf + sudo setenforce 0 || true #disable selinux + sudo yum -y install httpd + sudo service httpd start + sudo mkdir -p /var/www/$PUBLIC_HOSTNAME/public_html + sudo chmod -R oug+rwx /var/www + sudo chmod -R oug+rw /etc/httpd + sudo echo 'foobar' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html + sudo mkdir /etc/httpd/sites-available #letsencrypt requires this... + sudo mkdir /etc/httpd/sites-enabled #letsencrypt requires this... + #sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf + sudo echo """ + + ServerName $PUBLIC_HOSTNAME + DocumentRoot /var/www/$PUBLIC_HOSTNAME/public_html + ErrorLog /var/www/$PUBLIC_HOSTNAME/error.log + CustomLog /var/www/$PUBLIC_HOSTNAME/requests.log combined +""" >> /etc/httpd/conf.d/$PUBLIC_HOSTNAME.conf + #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/ +fi + +# run letsencrypt-apache2 via letsencrypt-auto +cd letsencrypt + +export SUDO=sudo +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO bootstrap/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO bootstrap/_rpm_common.sh +else + echo "Dont have bootstrapping for this OS!" + exit 1 +fi + +bootstrap/dev/venv.sh +sudo venv/bin/letsencrypt -v --debug --text --agree-dev-preview --agree-tos \ + --renew-by-default --redirect --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +if [ "$OS_TYPE" = "ubuntu" ] ; then + venv/bin/tox -e apacheconftest +else + echo Not running hackish apache tests on $OS_TYPE +fi + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +# return error if any of the subtests failed +if [ "$FAIL" = 1 ] ; then + exit 1 +fi diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh new file mode 100755 index 000000000..b7849755a --- /dev/null +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -0,0 +1,20 @@ +#!/bin/bash -xe + +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution + +cd letsencrypt +#git checkout v0.1.0 use --branch instead +SAVE="$PIP_EXTRA_INDEX_URL" +unset PIP_EXTRA_INDEX_URL +export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" +./letsencrypt-auto -v --debug --version +unset PIP_INDEX_URL + +export PIP_EXTRA_INDEX_URL="$SAVE" + +if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then + echo upgrade appeared to fail + exit 1 +fi +echo upgrade appeared to be successful diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh new file mode 100755 index 000000000..10d7c3b5e --- /dev/null +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -0,0 +1,15 @@ +#!/bin/bash -x + +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution + +# with curl, instance metadata available from EC2 metadata service: +#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) +#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) +#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) + +cd letsencrypt +./letsencrypt-auto certonly -v --standalone --debug \ + --text --agree-dev-preview --agree-tos \ + --renew-by-default --redirect \ + --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh new file mode 100755 index 000000000..234e70f68 --- /dev/null +++ b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh @@ -0,0 +1,7 @@ +#!/bin/bash -x + +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution + +cd letsencrypt +# help installs virtualenv and does nothing else +./letsencrypt-auto -v --debug --help all diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh new file mode 100755 index 000000000..f7f325d5c --- /dev/null +++ b/tests/letstest/scripts/test_tox.sh @@ -0,0 +1,80 @@ +#!/bin/bash -x +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="venv" +# The path to the letsencrypt-auto script. Everything that uses these might +# at some point be inlined... +LEA_PATH=./letsencrypt/ +VENV_PATH=${LEA_PATH/$VENV_NAME} +VENV_BIN=${VENV_PATH}/bin +BOOTSTRAP=${LEA_PATH}/bootstrap + +SUDO=sudo + +ExperimentalBootstrap() { + # Arguments: Platform name, boostrap script name, SUDO command (iff needed) + if [ "$2" != "" ] ; then + echo "Bootstrapping dependencies for $1..." + if [ "$3" != "" ] ; then + "$3" "$BOOTSTRAP/$2" + else + "$BOOTSTRAP/$2" + fi + fi +} + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -f $BOOTSTRAP/debian.sh ] ; then + echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" + exit 1 +fi + +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO $BOOTSTRAP/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO $BOOTSTRAP/_rpm_common.sh +elif `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + $SUDO $BOOTSTRAP/_suse_common.sh +elif [ -f /etc/arch-release ] ; then + if [ "$DEBUG" = 1 ] ; then + echo "Bootstrapping dependencies for Archlinux..." + $SUDO $BOOTSTRAP/archlinux.sh + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi +elif [ -f /etc/manjaro-release ] ; then + ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" +elif [ -f /etc/gentoo-release ] ; then + ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" +elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" +elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root +elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" +else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info" +fi +echo "Bootstrapped!" + +cd letsencrypt +./bootstrap/dev/venv.sh +PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + +if [ $PYVER -eq 26 ] ; then + venv/bin/tox -e py26 +else + venv/bin/tox -e py27 +fi diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml new file mode 100644 index 000000000..506225f86 --- /dev/null +++ b/tests/letstest/targets.yaml @@ -0,0 +1,99 @@ +targets: + #----------------------------------------------------------------------------- + #Ubuntu + - ami: ami-26d5af4c + name: ubuntu15.10 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-d92e6bb3 + name: ubuntu15.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-7b89cc11 + name: ubuntu14.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-9295d0f8 + name: ubuntu14.04LTS_32bit + type: ubuntu + virt: pv + user: ubuntu + - ami: ami-0611546c + name: ubuntu12.04LTS + type: ubuntu + virt: hvm + user: ubuntu + #----------------------------------------------------------------------------- + # Debian + - ami: ami-116d857a + name: debian8.1 + type: ubuntu + virt: hvm + user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + - ami: ami-e0efab88 + name: debian7.8.aws.1 + type: ubuntu + virt: hvm + user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + - ami: ami-e6eeaa8e + name: debian7.8.aws.1_32bit + type: ubuntu + virt: pv + user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + #----------------------------------------------------------------------------- + # Other Redhat Distros + - ami: ami-60b6c60a + name: amazonlinux-2015.09.1 + type: centos + virt: hvm + user: ec2-user + - ami: ami-0d4cfd66 + name: amazonlinux-2015.03.1 + type: centos + virt: hvm + user: ec2-user + - ami: ami-a8d369c0 + name: RHEL7 + type: centos + virt: hvm + user: ec2-user + - ami: ami-518bfb3b + name: fedora23 + type: centos + virt: hvm + user: fedora + #----------------------------------------------------------------------------- + # CentOS + # These Marketplace AMIs must, irritatingly, have their terms manually + # agreed to on the AWS marketplace site for any new AWS account using them... + - ami: ami-61bbf104 + name: centos7 + type: centos + virt: hvm + user: centos + # centos6 requires EPEL repo added + - ami: ami-57cd8732 + name: centos6 + type: centos + virt: hvm + user: centos + userdata: | + #cloud-config + runcmd: + - yum install -y epel-release + - iptables -F diff --git a/tools/release.sh b/tools/release.sh index eeabfd4a3..172f6fea1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -86,7 +86,7 @@ SetVersion() { done sed -i "s/^__version.*/__version__ = '$ver'/" letsencrypt/__init__.py - git add -p $SUBPKGS # interactive user input + git add -p letsencrypt $SUBPKGS # interactive user input } SetVersion "$version" git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" diff --git a/tox.ini b/tox.ini index d1fafe20f..dbd6d51fa 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ envlist = py26,py27,py33,py34,py35,cover,lint commands = pip install -e acme[testing] nosetests -v acme - pip install -r py26reqs.txt -e .[testing] + pip install -e .[testing] nosetests -v letsencrypt pip install -e letsencrypt-apache nosetests -v letsencrypt_apache @@ -67,3 +67,11 @@ commands = pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt + +[testenv:apacheconftest] +#basepython = python2.7 +setenv = + LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/apacheconftest/bin/letsencrypt +commands = + pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test --debian-modules