diff --git a/.gitignore b/.gitignore index ba843d9cc..1becea3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ letsencrypt.log # auth --cert-path --chain-path /*.pem + +# letstest +tests/letstest/letest-*/ +tests/letstest/*.pem diff --git a/Dockerfile b/Dockerfile index 02aa0f0d7..da0110604 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,6 @@ COPY letsencrypt-apache /opt/letsencrypt/src/letsencrypt-apache/ COPY letsencrypt-nginx /opt/letsencrypt/src/letsencrypt-nginx/ -# py26reqs.txt not installed! RUN virtualenv --no-site-packages -p python2 /opt/letsencrypt/venv && \ /opt/letsencrypt/venv/bin/pip install \ -e /opt/letsencrypt/src/acme \ diff --git a/Dockerfile-dev b/Dockerfile-dev index b89411c90..3c5b53966 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -32,7 +32,6 @@ RUN /opt/letsencrypt/src/ubuntu.sh && \ # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -# py26reqs.txt not installed! COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini pep8.travis.sh .pep8 .pylintrc /opt/letsencrypt/src/ # all above files are necessary for setup.py, however, package source diff --git a/MANIFEST.in b/MANIFEST.in index a82c7dd8c..a6f9ae2b6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include py26reqs.txt include README.rst include CHANGES.rst include CONTRIBUTING.md diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index c38cea414..0f5f0e4bd 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -1,12 +1,12 @@ """ACME protocol implementation. This module is an implementation of the `ACME protocol`_. Latest -supported version: `v02`_. +supported version: `draft-ietf-acme-01`_. -.. _`ACME protocol`: https://github.com/letsencrypt/acme-spec +.. _`ACME protocol`: https://github.com/ietf-wg-acme/acme/ -.. _`v02`: - https://github.com/letsencrypt/acme-spec/commit/d328fea2d507deb9822793c512830d827a4150c4 +.. _`draft-ietf-acme-01`: + https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ diff --git a/acme/setup.py b/acme/setup.py index 5fc1bd8d0..3ad932448 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -10,8 +10,6 @@ install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', - 'ndg-httpsclient', # urllib3 InsecurePlatformWarning (#304) - 'pyasn1', # urllib3 InsecurePlatformWarning (#304) # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13', 'pyrfc3339', @@ -32,6 +30,11 @@ if sys.version_info < (2, 7): else: install_requires.append('mock') +if sys.version_info < (2, 7, 9): + # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) + install_requires.append('ndg-httpsclient') + install_requires.append('pyasn1') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/bootstrap/README b/bootstrap/README index 89fd8b6ba..d91780903 100644 --- a/bootstrap/README +++ b/bootstrap/README @@ -2,6 +2,5 @@ This directory contains scripts that install necessary OS-specific prerequisite dependencies (see docs/using.rst). General dependencies: -- git-core: py26reqs.txt git+https://* - ca-certificates: communication with demo ACMO server at - https://www.letsencrypt-demo.org, py26reqs.txt git+https://* + https://www.letsencrypt-demo.org diff --git a/bootstrap/_arch_common.sh b/bootstrap/_arch_common.sh index f66067ffb..2b512792f 100755 --- a/bootstrap/_arch_common.sh +++ b/bootstrap/_arch_common.sh @@ -8,7 +8,6 @@ # ./bootstrap/dev/_common_venv.sh deps=" - git python2 python-virtualenv gcc diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 4c6b91a33..8b96fe6f1 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -24,26 +24,56 @@ apt-get update # distro version (#346) virtualenv= -if apt-cache show virtualenv > /dev/null ; then +if apt-cache show virtualenv > /dev/null 2>&1; then virtualenv="virtualenv" fi -if apt-cache show python-virtualenv > /dev/null ; then +if apt-cache show python-virtualenv > /dev/null 2>&1; then virtualenv="$virtualenv python-virtualenv" fi +augeas_pkg=libaugeas0 +AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + +if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then + /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." + sleep 1s + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + + echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list + apt-get update + fi + fi + apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 + augeas_pkg= + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs +fi + apt-get install -y --no-install-recommends \ - git \ python \ python-dev \ $virtualenv \ gcc \ dialog \ - libaugeas0 \ + $augeas_pkg \ libssl-dev \ libffi-dev \ ca-certificates \ + + if ! command -v virtualenv > /dev/null ; then echo Failed to install a working \"virtualenv\" command, exiting exit 1 diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh index a718db7ff..f49dc00f0 100755 --- a/bootstrap/_gentoo_common.sh +++ b/bootstrap/_gentoo_common.sh @@ -1,6 +1,6 @@ #!/bin/sh -PACKAGES="dev-vcs/git +PACKAGES=" dev-lang/python:2.7 dev-python/virtualenv dev-util/dialog diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh index 411d7bd92..92b54b720 100755 --- a/bootstrap/_rpm_common.sh +++ b/bootstrap/_rpm_common.sh @@ -33,9 +33,7 @@ then fi fi -# "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails) if ! $tool install -y \ - git-core \ gcc \ dialog \ augeas-libs \ diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh index 46f9d693b..efeebe4f8 100755 --- a/bootstrap/_suse_common.sh +++ b/bootstrap/_suse_common.sh @@ -2,7 +2,7 @@ # SLE12 don't have python-virtualenv -zypper -nq in -l git-core \ +zypper -nq in -l \ python \ python-devel \ python-virtualenv \ diff --git a/bootstrap/dev/venv.sh b/bootstrap/dev/venv.sh index 2bd32a89b..11ab417dd 100755 --- a/bootstrap/dev/venv.sh +++ b/bootstrap/dev/venv.sh @@ -4,7 +4,6 @@ export VENV_ARGS="--python python2" ./bootstrap/dev/_venv_common.sh \ - -r py26reqs.txt \ -e acme[testing] \ -e .[dev,docs,testing] \ -e letsencrypt-apache \ diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh index 180ee21b4..4482c35cd 100755 --- a/bootstrap/freebsd.sh +++ b/bootstrap/freebsd.sh @@ -1,7 +1,6 @@ #!/bin/sh -xe pkg install -Ay \ - git \ python \ py27-virtualenv \ augeas \ diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh index ff1a50c6c..5042178d9 100755 --- a/bootstrap/venv.sh +++ b/bootstrap/venv.sh @@ -20,7 +20,7 @@ fi pip install -U setuptools pip install -U pip -pip install -U -r py26reqs.txt letsencrypt letsencrypt-apache # letsencrypt-nginx +pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx echo echo "Congratulations, Let's Encrypt has been successfully installed/updated!" diff --git a/docs/using.rst b/docs/using.rst index 1423d6eba..5da13f02c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -64,7 +64,7 @@ or for full help, type: ``letsencrypt-auto`` is the recommended method of running the Let's Encrypt client beta releases on systems that don't have a packaged version. Debian, -Arch linux and FreeBSD now have native packages, so on those +Arch linux, FreeBSD, and OpenBSD now have native packages, so on those systems you can just install ``letsencrypt`` (and perhaps ``letsencrypt-apache``). If you'd like to run the latest copy from Git, or run your own locally modified copy of the client, follow the instructions in @@ -351,6 +351,11 @@ Operating System Packages * Port: ``cd /usr/ports/security/py-letsencrypt && make install clean`` * Package: ``pkg install py27-letsencrypt`` +**OpenBSD** + + * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` + * Package: ``pkg_add letsencrypt`` + **Arch Linux** .. code-block:: shell @@ -366,7 +371,7 @@ If you run Debian Stretch or Debian Sid, you can install letsencrypt packages. sudo apt-get update sudo apt-get install letsencrypt python-letsencrypt-apache -If you don't want to use the Apache plugin, you can ommit the +If you don't want to use the Apache plugin, you can omit the ``python-letsencrypt-apache`` package. Packages for Debian Jessie are coming in the next few weeks. diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index d665ea7a7..0f2cb7b45 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -59,7 +59,7 @@ let empty = Util.empty_dos let indent = Util.indent (* borrowed from shellvars.aug *) -let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ '"\t\r\n])|\\\\"|\\\\'/ +let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index f72492ac2..39b8f2426 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -542,7 +542,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if "ssl_module" not in self.parser.modules: self.enable_mod("ssl", temp=temp) - + if self.version >= (2, 4) and "socache_shmcb_module" not in self.parser.modules: + self.enable_mod("socache_shmcb", temp=temp) # Check for Listen # Note: This could be made to also look for ip:443 combo listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] @@ -1399,7 +1400,7 @@ def _get_mod_deps(mod_name): """ deps = { - "ssl": ["setenvif", "mime", "socache_shmcb"] + "ssl": ["setenvif", "mime"] } return deps.get(mod_name, []) diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index ec5211ae4..418e0ec39 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -19,7 +19,6 @@ class ApacheParser(object): :ivar str root: Normalized absolute path to the server root directory. Without trailing slash. - :ivar str root: Server root :ivar set modules: All module names that are currently enabled. :ivar dict loc: Location to place directives, root - configuration origin, default - user config file, name - NameVirtualHost, @@ -36,6 +35,7 @@ class ApacheParser(object): # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine # This only handles invocation parameters and Define directives! self.variables = {} + self.unparsable = False self.update_runtime_variables(ctl) self.aug = aug @@ -60,6 +60,11 @@ class ApacheParser(object): # Sites-available is not included naturally in configuration self._parse_file(os.path.join(self.root, "sites-available") + "/*") + #check to see if there were unparsed define statements + if self.unparsable: + if self.find_dir("Define", exclude=False): + raise errors.PluginError("Error parsing runtime variables") + def init_modules(self): """Iterates on the configuration until no new modules are loaded. @@ -101,7 +106,8 @@ class ApacheParser(object): try: matches.remove("DUMP_RUN_CFG") except ValueError: - raise errors.PluginError("Unable to parse runtime variables") + self.unparsable = True + return for match in matches: if match.count("=") > 1: diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index f2bf89d2c..617ff96af 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -28,10 +28,21 @@ class TwoVhost80Test(util.ApacheTest): self.config = util.get_apache_configurator( self.config_path, self.config_dir, self.work_dir) - + self.config = self.mock_deploy_cert(self.config) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/two_vhost_80") + def mock_deploy_cert(self, config): + """A test for a mock deploy cert""" + self.config.real_deploy_cert = self.config.deploy_cert + def mocked_deploy_cert(*args, **kwargs): + """a helper to mock a deployed cert""" + with mock.patch( + "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + config.real_deploy_cert(*args, **kwargs) + self.config.deploy_cert = mocked_deploy_cert + return self.config + def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) @@ -251,6 +262,7 @@ class TwoVhost80Test(util.ApacheTest): # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] + self.config = self.mock_deploy_cert(self.config) self.config.deploy_cert( "random.demo", "example/cert.pem", "example/key.pem", "example/cert_chain.pem", "example/fullchain.pem") @@ -277,6 +289,7 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl_no_fullchain(self): self.config = util.get_apache_configurator( self.config_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -290,6 +303,7 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_old_apache_no_chain(self): self.config = util.get_apache_configurator( self.config_path, self.config_dir, self.work_dir, version=(2, 4, 7)) + self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index bc1f316f9..352c2fcf4 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -151,8 +151,8 @@ class BasicParserTest(util.ParserTest): @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_vars_bad_output(self, mock_cfg): mock_cfg.return_value = "Define: TLS=443=24" - self.assertRaises( - errors.PluginError, self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables("ctl") + self.assertTrue(self.parser.unparsable) mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24" self.assertRaises( @@ -185,6 +185,21 @@ class ParserInitTest(util.ApacheTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") + def test_unparsable(self, mock_cfg): + from letsencrypt_apache.parser import ApacheParser + def unparsable_true(self, arg): + """a helper to set the self unparsabale to true""" + print "side effect has passed in arg: %s", arg + self.unparsable = True + with mock.patch.object(ApacheParser, 'update_runtime_variables', autospec=True) as urv: + urv.side_effect = unparsable_true + mock_cfg.return_value = ('Define: TEST') + self.assertRaises( + errors.PluginError, + ApacheParser, self.aug, os.path.relpath(self.config_path), "ctl") + self.assertEquals(1, 1) + def test_root_normalized(self): from letsencrypt_apache.parser import ApacheParser diff --git a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py index f4dff7734..7db4eee6f 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py @@ -78,7 +78,9 @@ class TlsSniPerformTest(util.ApacheTest): # pylint: disable=protected-access self.sni._setup_challenge_cert = mock_setup_cert - sni_responses = self.sni.perform() + with mock.patch( + "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + sni_responses = self.sni.perform() self.assertEqual(mock_setup_cert.call_count, 2) diff --git a/letsencrypt-auto b/letsencrypt-auto index 44c71883c..20465dbb1 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -47,13 +47,13 @@ if test "`id -u`" -ne "0" ; then args="" # This `while` loop iterates over all parameters given to this function. # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrap in a pair of `'`, then append to `$args` string + # will be wrapped in a pair of `'`, then appended to `$args` string # For example, `echo "It's only 1\$\!"` will be escaped to: # 'echo' 'It'"'"'s only 1$!' # │ │└┼┘│ # │ │ │ └── `'s only 1$!'` the literal string # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings followed it + # │ └── `'It'`, to be concatenated with the strings following it # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself while [ $# -ne 0 ]; do args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " @@ -97,6 +97,7 @@ DeterminePythonVersion() { export LE_PYTHON=${LE_PYTHON:-python} else echo "Cannot find any Pythons... please install one!" + exit 1 fi PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` @@ -175,7 +176,7 @@ if [ "$VERBOSE" = 1 ] ; then echo $VENV_BIN/pip install -U setuptools $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -r "$LEA_PATH"/py26reqs.txt -U letsencrypt letsencrypt-apache + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache # nginx is buggy / disabled for now, but upgrade it if the user has # installed it manually if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then @@ -187,8 +188,6 @@ else $VENV_BIN/pip install -U pip > /dev/null printf . # nginx is buggy / disabled for now... - $VENV_BIN/pip install -r "$LEA_PATH"/py26reqs.txt > /dev/null - printf . $VENV_BIN/pip install -U letsencrypt > /dev/null printf . $VENV_BIN/pip install -U letsencrypt-apache > /dev/null @@ -201,5 +200,5 @@ fi # Explain what's about to happen, for the benefit of those getting sudo # password prompts... -echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" +echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" $SUDO $VENV_BIN/letsencrypt "$@" diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index dfebde13c..89606089f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1189,7 +1189,7 @@ def _plugins_parsing(helpful, plugins): # These would normally be a flag within the webroot plugin, but because # they are parsed in conjunction with --domains, they live here for - # legibiility. helpful.add_plugin_ags must be called first to add the + # legibility. helpful.add_plugin_ags must be called first to add the # "webroot" topic helpful.add("webroot", "-w", "--webroot-path", action=WebrootPathProcessor, help="public_html / webroot path. This can be specified multiple times to " diff --git a/letsencrypt/plugins/webroot_test.py b/letsencrypt/plugins/webroot_test.py index 9f5b6bba8..defe9396b 100644 --- a/letsencrypt/plugins/webroot_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -66,8 +66,16 @@ class AuthenticatorTest(unittest.TestCase): def test_prepare_reraises_other_errors(self): self.auth.full_path = os.path.join(self.path, "null") + permission_canary = os.path.join(self.path, "rnd") + with open(permission_canary, "w") as f: + f.write("thingimy") os.chmod(self.path, 0o000) - self.assertRaises(errors.PluginError, self.auth.prepare) + try: + open(permission_canary, "r") + print "Warning, running tests as root skips permissions tests..." + except IOError: + # ok, permissions work, test away... + self.assertRaises(errors.PluginError, self.auth.prepare) os.chmod(self.path, 0o700) @mock.patch("letsencrypt.plugins.webroot.os.chown") diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index ed4910ef9..2d2675745 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -116,6 +116,7 @@ def renew(cert, old_version): def _cli_log_handler(args, level, fmt): # pylint: disable=unused-argument handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) + handler.setLevel(level) return handler @@ -181,7 +182,9 @@ def main(cli_args=sys.argv[1:]): # RenewableCert object for this cert at all, which could # dramatically improve performance for large deployments # where autorenewal is widely turned off. - cert = storage.RenewableCert(renewal_file, cli_config) + cert = storage.RenewableCert( + os.path.join(cli_config.renewal_configs_dir, renewal_file), + cli_config) except errors.CertStorageError: # This indicates an invalid renewal configuration file, such # as one missing a required parameter (in the future, perhaps diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 3b2b548b0..ac71bd9fe 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -260,7 +260,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :returns: The path to the current version of the specified member. - :rtype: str + :rtype: str or None """ if kind not in ALL_FOUR: @@ -450,12 +450,15 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :param int version: the desired version number :returns: the subject names :rtype: `list` of `str` + :raises .CertStorageError: if could not find cert file. """ if version is None: target = self.current_target("cert") else: target = self.version("cert", version) + if target is None: + raise errors.CertStorageError("could not find cert file") with open(target) as f: return crypto_util.get_sans_from_cert(f.read()) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 269a9193d..61a9a6e75 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -767,6 +767,8 @@ class RenewableCertTests(BaseRenewableCertTest): def test_bad_config_file(self): from letsencrypt import renewer + os.unlink(os.path.join(self.cli_config.renewal_configs_dir, + "example.org.conf")) with open(os.path.join(self.cli_config.renewal_configs_dir, "bad.conf"), "w") as f: f.write("incomplete = configfile\n") diff --git a/py26reqs.txt b/py26reqs.txt deleted file mode 100644 index a94b22c0c..000000000 --- a/py26reqs.txt +++ /dev/null @@ -1,2 +0,0 @@ -# https://github.com/bw2/ConfigArgParse/issues/17 -git+https://github.com/kuba/ConfigArgParse.git@python2.6-0.9.3#egg=ConfigArgParse diff --git a/setup.py b/setup.py index 40c6ac16c..ae36777ef 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ version = meta['version'] install_requires = [ 'acme=={0}'.format(version), - 'ConfigArgParse', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', @@ -53,10 +52,19 @@ if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', + 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 'mock<1.1.0', ]) else: - install_requires.append('mock') + install_requires.extend([ + 'ConfigArgParse', + 'mock', + ]) + +if sys.version_info < (2, 7, 9): + # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) + install_requires.append('ndg-httpsclient') + install_requires.append('pyasn1') dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 diff --git a/tests/apache-conf-files/passing/graphite-quote-1934.conf b/tests/apache-conf-files/passing/graphite-quote-1934.conf new file mode 100644 index 000000000..2a8734b43 --- /dev/null +++ b/tests/apache-conf-files/passing/graphite-quote-1934.conf @@ -0,0 +1,21 @@ + + + WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=_graphite group=_graphite + WSGIProcessGroup _graphite + WSGIImportScript /usr/share/graphite-web/graphite.wsgi process-group=_graphite application-group=%{GLOBAL} + WSGIScriptAlias / /usr/share/graphite-web/graphite.wsgi + + Alias /content/ /usr/share/graphite-web/static/ + + SetHandler None + + + ErrorLog ${APACHE_LOG_DIR}/graphite-web_error.log + + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/graphite-web_access.log combined + + diff --git a/tests/apache-conf-files/passing/rewrite-quote-1960.conf b/tests/apache-conf-files/passing/rewrite-quote-1960.conf new file mode 100644 index 000000000..26214e7b0 --- /dev/null +++ b/tests/apache-conf-files/passing/rewrite-quote-1960.conf @@ -0,0 +1,7 @@ + + RewriteEngine On + RewriteCond %{REQUEST_URI} ^.*(,|;|:|<|>|">|"<|/|\\\.\.\\).* [NC,OR] + RewriteCond %{REQUEST_URI} ^.*(\=|\@|\[|\]|\^|\`|\{|\}|\~).* [NC,OR] + RewriteCond %{REQUEST_URI} ^.*(\'|%0A|%0D|%27|%3C|%3E|%00).* [NC] + RewriteRule ^(.*)$ - [F,L] + diff --git a/tests/letstest/README.md b/tests/letstest/README.md new file mode 100644 index 000000000..a085e9d91 --- /dev/null +++ b/tests/letstest/README.md @@ -0,0 +1,44 @@ +# letstest +simple aws testfarm scripts for letsencrypt client testing + +- Configures (canned) boulder server +- Launches EC2 instances with a given list of AMIs for different distros +- Copies letsencrypt repo and puts it on the instances +- Runs letsencrypt tests (bash scripts) on all of these +- Logs execution and success/fail for debugging + +## Notes + - Some AWS images, e.g. official CentOS and FreeBSD images + require acceptance of user terms on the AWS marketplace + website. This can't be automated. + - AWS EC2 has a default limit of 20 t2/t1 instances, if more + are needed, they need to be requested via online webform. + +## Usage + - Requires AWS IAM secrets to be set up with aws cli + - Requires an AWS associated keyfile .pem + +``` +>aws configure --profile HappyHacker +[interactive: enter secrets for IAM role] +>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair --query 'KeyMaterial' --output text > MyKeyPair.pem +``` +then: +``` +>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_apache2.sh +``` + +## Scripts +example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed +to them at runtime via environment variables. test_apache2.sh is a useful reference. + +Note that the 
test_letsencrypt_auto_*
scripts pull code from PyPI using the letsencrypt-auto script, +__not__ the local python code. test_apache2 runs the dev venv and does local tests. + +see: +- https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html +- https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html + +main repos: +- https://github.com/letsencrypt/boulder +- https://github.com/letsencrypt/letsencrypt diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml new file mode 100644 index 000000000..e707b8636 --- /dev/null +++ b/tests/letstest/apache2_targets.yaml @@ -0,0 +1,57 @@ +targets: + #----------------------------------------------------------------------------- + # Apache 2.4 + - ami: ami-26d5af4c + name: ubuntu15.10 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-d92e6bb3 + name: ubuntu15.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-7b89cc11 + name: ubuntu14.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-9295d0f8 + name: ubuntu14.04LTS_32bit + type: ubuntu + virt: pv + user: ubuntu + - ami: ami-116d857a + name: debian8.1 + type: debian + virt: hvm + user: admin + userdata: | + #cloud-init + runcmd: + - [ apt-get, install, -y, curl ] + #----------------------------------------------------------------------------- + # Apache 2.2 + # - ami: ami-0611546c + # name: ubuntu12.04LTS + # type: ubuntu + # virt: hvm + # user: ubuntu + # - ami: ami-e0efab88 + # name: debian7.8.aws.1 + # type: debian + # virt: hvm + # user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + # - ami: ami-e6eeaa8e + # name: debian7.8.aws.1_32bit + # type: debian + # virt: pv + # user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] \ No newline at end of file diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py new file mode 100644 index 000000000..fb29ab9eb --- /dev/null +++ b/tests/letstest/multitester.py @@ -0,0 +1,492 @@ +""" +Letsencrypt Integration Test Tool + +- Configures (canned) boulder server +- Launches EC2 instances with a given list of AMIs for different distros +- Copies letsencrypt repo and puts it on the instances +- Runs letsencrypt tests (bash scripts) on all of these +- Logs execution and success/fail for debugging + +Notes: + - Some AWS images, e.g. official CentOS and FreeBSD images + require acceptance of user terms on the AWS marketplace + website. This can't be automated. + - AWS EC2 has a default limit of 20 t2/t1 instances, if more + are needed, they need to be requested via online webform. + +Usage: + - Requires AWS IAM secrets to be set up with aws cli + - Requires an AWS associated keyfile .pem + +>aws configure --profile HappyHacker +[interactive: enter secrets for IAM role] +>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \ + --query 'KeyMaterial' --output text > MyKeyPair.pem +then: +>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh +see: + https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html + https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html +""" + +from __future__ import print_function +from __future__ import with_statement + +import sys, os, time, argparse, socket +import multiprocessing as mp +from multiprocessing import Manager +import urllib2 +import yaml +import boto3 +import fabric +from fabric.api import run, execute, local, env, sudo, cd, lcd +from fabric.operations import get, put +from fabric.context_managers import shell_env + +# Command line parser +#------------------------------------------------------------------------------- +parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.') +parser.add_argument('config_file', + help='yaml configuration file for AWS server cluster') +parser.add_argument('key_file', + help='key file (.pem) for AWS') +parser.add_argument('aws_profile', + help='profile for AWS (i.e. as in ~/.aws/certificates)') +parser.add_argument('test_script', + default='test_letsencrypt_auto_certonly_standalone.sh', + help='path of bash script in to deploy and run') +#parser.add_argument('--script_args', +# nargs='+', +# help='space-delimited list of arguments to pass to the bash test script', +# required=False) +parser.add_argument('--repo', + default='https://github.com/letsencrypt/letsencrypt.git', + help='letsencrypt git repo to use') +parser.add_argument('--branch', + default='~', + help='letsencrypt git branch to trial') +parser.add_argument('--pull_request', + default='~', + help='letsencrypt/letsencrypt pull request to trial') +parser.add_argument('--merge_master', + action='store_true', + help="if set merges PR into master branch of letsencrypt/letsencrypt") +parser.add_argument('--saveinstances', + action='store_true', + help="don't kill EC2 instances after run, useful for debugging") +parser.add_argument('--alt_pip', + default='https://certainly.isnot.org/pip/', + help="server from which to pull candidate release packages") +cl_args = parser.parse_args() + +# Credential Variables +#------------------------------------------------------------------------------- +# assumes naming: = .pem +KEYFILE = cl_args.key_file +KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0] +PROFILE = cl_args.aws_profile + +# Globals +#------------------------------------------------------------------------------- +BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1 +LOGDIR = "" #points to logging / working directory +# boto3/AWS api globals +AWS_SESSION = None +EC2 = None + +# Boto3/AWS automation functions +#------------------------------------------------------------------------------- +def make_security_group(): + # will fail if security group of GroupName already exists + # cannot have duplicate SGs of the same name + mysg = EC2.create_security_group(GroupName="letsencrypt_test", + Description='security group for automated testing') + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443) + # for boulder wfe (http) server + mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000) + # for mosh + mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000) + return mysg + +def make_instance(instance_name, + ami_id, + keyname, + machine_type='t2.micro', + security_groups=['letsencrypt_test'], + userdata=""): #userdata contains bash or cloud-init script + + new_instance = EC2.create_instances( + ImageId=ami_id, + SecurityGroups=security_groups, + KeyName=keyname, + MinCount=1, + MaxCount=1, + UserData=userdata, + InstanceType=machine_type)[0] + + # brief pause to prevent rare error on EC2 delay, should block until ready instead + time.sleep(1.0) + + # give instance a name + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + return new_instance + +def terminate_and_clean(instances): + """ + Some AMIs specify EBS stores that won't delete on instance termination. + These must be manually deleted after shutdown. + """ + volumes_to_delete = [] + for instance in instances: + for bdmap in instance.block_device_mappings: + if 'Ebs' in bdmap.keys(): + if not bdmap['Ebs']['DeleteOnTermination']: + volumes_to_delete.append(bdmap['Ebs']['VolumeId']) + + for instance in instances: + instance.terminate() + + # can't delete volumes until all attaching instances are terminated + _ids = [instance.id for instance in instances] + all_terminated = False + while not all_terminated: + all_terminated = True + for _id in _ids: + # necessary to reinit object for boto3 to get true state + inst = EC2.Instance(id=_id) + if inst.state['Name'] != 'terminated': + all_terminated = False + time.sleep(5) + + for vol_id in volumes_to_delete: + volume = EC2.Volume(id=vol_id) + volume.delete() + + return volumes_to_delete + + +# Helper Routines +#------------------------------------------------------------------------------- +def block_until_http_ready(urlstring, wait_time=10, timeout=240): + "Blocks until server at urlstring can respond to http requests" + server_ready = False + t_elapsed = 0 + while not server_ready and t_elapsed < timeout: + try: + sys.stdout.write('.') + sys.stdout.flush() + req = urllib2.Request(urlstring) + response = urllib2.urlopen(req) + #if response.code == 200: + server_ready = True + except urllib2.URLError: + pass + time.sleep(wait_time) + t_elapsed += wait_time + +def block_until_ssh_open(ipstring, wait_time=10, timeout=120): + "Blocks until server at ipstring has an open port 22" + reached = False + t_elapsed = 0 + while not reached and t_elapsed < timeout: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((ipstring, 22)) + reached = True + except socket.error as err: + time.sleep(wait_time) + t_elapsed += wait_time + sock.close() + +def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20): + "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections" + # the reinstantiation from id is necessary to force boto3 + # to correctly update the 'state' variable during init + _id = booting_instance.id + _instance = EC2.Instance(id=_id) + _state = _instance.state['Name'] + _ip = _instance.public_ip_address + while _state != 'running' or _ip is None: + time.sleep(wait_time) + _instance = EC2.Instance(id=_id) + _state = _instance.state['Name'] + _ip = _instance.public_ip_address + block_until_ssh_open(_ip) + time.sleep(extra_wait_time) + return _instance + + +# Fabric Routines +#------------------------------------------------------------------------------- +def local_git_clone(repo_url): + "clones master of repo_url" + with lcd(LOGDIR): + local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') + local('git clone %s'% repo_url) + local('tar czf le.tar.gz letsencrypt') + +def local_git_branch(repo_url, branch_name): + "clones branch of repo_url" + with lcd(LOGDIR): + local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') + local('git clone %s --branch %s --single-branch'%(repo_url, branch_name)) + local('tar czf le.tar.gz letsencrypt') + +def local_git_PR(repo_url, PRnumstr, merge_master=True): + "clones specified pull request from repo_url and optionally merges into master" + with lcd(LOGDIR): + local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') + local('git clone %s'% repo_url) + local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) + local('cd letsencrypt && git co lePRtest') + if merge_master: + local('cd letsencrypt && git remote update origin') + local('cd letsencrypt && git merge origin/master -m "testmerge"') + local('tar czf le.tar.gz letsencrypt') + +def local_repo_to_remote(): + "copies local tarball of repo to remote" + with lcd(LOGDIR): + put(local_path='le.tar.gz', remote_path='') + run('tar xzf le.tar.gz') + +def local_repo_clean(): + "delete tarball" + with lcd(LOGDIR): + local('rm le.tar.gz') + +def deploy_script(scriptpath, *args): + "copies to remote and executes local script" + #with lcd('scripts'): + put(local_path=scriptpath, remote_path='', mirror_local_mode=True) + scriptfile = os.path.split(scriptpath)[1] + args_str = ' '.join(args) + run('./'+scriptfile+' '+args_str) + +def run_boulder(): + with cd('$GOPATH/src/github.com/letsencrypt/boulder'): + run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost') + run('nohup ./start.py >& /dev/null < /dev/null &') + +def config_and_launch_boulder(instance): + execute(deploy_script, 'scripts/boulder_config.sh') + execute(run_boulder) + +def install_and_launch_letsencrypt(instance, boulder_url, target): + execute(local_repo_to_remote) + with shell_env(BOULDER_URL=boulder_url, + PUBLIC_IP=instance.public_ip_address, + PRIVATE_IP=instance.private_ip_address, + PUBLIC_HOSTNAME=instance.public_dns_name, + PIP_EXTRA_INDEX_URL=cl_args.alt_pip, + OS_TYPE=target['type']): + execute(deploy_script, cl_args.test_script) + +def grab_letsencrypt_log(): + "grabs letsencrypt.log via cat into logged stdout" + sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \ + cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi') + # fallback file if /var/log is unwriteable...? correct? + sudo('if [ -f ./letsencrypt.log ]; then \ + cat ./letsencrypt.log; else echo "[nolocallog]"; fi') + +#------------------------------------------------------------------------------- +# SCRIPT BEGINS +#------------------------------------------------------------------------------- + +# Fabric library controlled through global env parameters +env.key_filename = KEYFILE +env.shell = '/bin/bash -l -i -c' +env.connection_attempts = 5 +env.timeout = 10 +# replace default SystemExit thrown by fabric during trouble +class FabricException(Exception): + pass +env['abort_exception'] = FabricException + +# Set up local copy of git repo +#------------------------------------------------------------------------------- +LOGDIR = "letest-%d"%int(time.time()) +print("Making local dir for test repo and logs: %s"%LOGDIR) +local('mkdir %s'%LOGDIR) + +# figure out what git object to test and locally create it in LOGDIR +print("Making local git repo") +try: + if cl_args.pull_request != '~': + print('Testing PR %s '%cl_args.pull_request, + "MERGING into master" if cl_args.merge_master else "") + execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master) + elif cl_args.branch != '~': + print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo)) + execute(local_git_branch, cl_args.repo, cl_args.branch) + else: + print('Testing master of %s'%cl_args.repo) + execute(local_git_clone, cl_args.repo) +except FabricException: + print("FAIL: trouble with git repo") + exit() + + +# Set up EC2 instances +#------------------------------------------------------------------------------- +configdata = yaml.load(open(cl_args.config_file, 'r')) +targetlist = configdata['targets'] +print('Testing against these images: [%d total]'%len(targetlist)) +for target in targetlist: + print(target['ami'], target['name']) + +print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE)) +AWS_SESSION = boto3.session.Session(profile_name=PROFILE) +EC2 = AWS_SESSION.resource('ec2') + +print("Making Security Group") +sg_exists = False +for sg in EC2.security_groups.all(): + if sg.group_name == 'letsencrypt_test': + sg_exists = True + print(" %s already exists"%'letsencrypt_test') +if not sg_exists: + make_security_group() + time.sleep(30) + +print("Requesting Instances...") +boulder_server = make_instance('le-boulderserver', + BOULDER_AMI, + KEYNAME, + #machine_type='t2.micro', + machine_type='t2.medium', + security_groups=['letsencrypt_test']) + +instances = [] +for target in targetlist: + if target['virt'] == 'hvm': + machine_type = 't2.micro' + else: + machine_type = 't1.micro' + if 'userdata' in target.keys(): + userdata = target['userdata'] + else: + userdata = '' + instances.append(make_instance('le-%s'%target['name'], + target['ami'], + KEYNAME, + machine_type=machine_type, + userdata=userdata)) + + +# Configure and launch boulder server +#------------------------------------------------------------------------------- +print("Waiting on Boulder Server") +boulder_server = block_until_instance_ready(boulder_server) +print(" server %s"%boulder_server) + +print("Configuring and Launching Boulder") + +# env.host_string defines the ssh user and host for connection +env.host_string = "ubuntu@%s"%boulder_server.public_ip_address +print("Boulder Server at (SSH):", env.host_string) +config_and_launch_boulder(boulder_server) +# blocking often unnecessary, but cheap EC2 VMs can get very slow +block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, + timeout=500) + +boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address +print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) +print("Boulder Server at (EC2 private ip): %s"%boulder_url) + +# Install and launch client scripts in parallel +#------------------------------------------------------------------------------- +print("Uploading and running test script in parallel: %s"%cl_args.test_script) +print("Output routed to log files in %s"%LOGDIR) +# (Advice: always use Manager.Queue, never regular multiprocessing.Queue +# the latter has implementation flaws that deadlock it in some circumstances) +manager = Manager() +outqueue = manager.Queue() +inqueue = manager.Queue() +SENTINEL = None #queue kill signal + +# launch as many processes as clients to test +num_processes = len(targetlist) +jobs = [] #keep a reference to current procs + +def test_client_process(inqueue, outqueue): + cur_proc = mp.current_process() + for inreq in iter(inqueue.get, SENTINEL): + ii, target = inreq + + #save all stdout to log file + sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w') + + print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) + instances[ii] = block_until_instance_ready(instances[ii]) + print("server %s at %s"%(instances[ii], instances[ii].public_ip_address)) + env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address) + print(env.host_string) + + try: + install_and_launch_letsencrypt(instances[ii], boulder_url, target) + outqueue.put((ii, target, 'pass')) + print("%s - %s SUCCESS"%(target['ami'], target['name'])) + except: + outqueue.put((ii, target, 'fail')) + print("%s - %s FAIL"%(target['ami'], target['name'])) + pass + + # append server letsencrypt.log to each per-machine output log + print("\n\nletsencrypt.log\n" + "-"*80 + "\n") + try: + execute(grab_letsencrypt_log) + except: + print("log fail\n") + pass + +# initiate process execution +for i in range(num_processes): + p = mp.Process(target=test_client_process, args=(inqueue, outqueue)) + jobs.append(p) + p.daemon = True # kills subprocesses if parent is killed + p.start() + +# fill up work queue +for ii, target in enumerate(targetlist): + inqueue.put((ii, target)) + +# add SENTINELs to end client processes +for i in range(num_processes): + inqueue.put(SENTINEL) +# wait on termination of client processes +for p in jobs: + p.join() +# add SENTINEL to output queue +outqueue.put(SENTINEL) + +# clean up +execute(local_repo_clean) + +# print and save summary results +results_file = open(LOGDIR+'/results', 'w') +outputs = [outq for outq in iter(outqueue.get, SENTINEL)] +outputs.sort(key=lambda x: x[0]) +for outq in outputs: + ii, target, status = outq + print('%d %s %s'%(ii, target['name'], status)) + results_file.write('%d %s %s\n'%(ii, target['name'], status)) +results_file.close() + +if not cl_args.saveinstances: + print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') + boulder_server.terminate() + terminate_and_clean(instances) +else: + # print login information for the boxes for debugging + for ii, target in enumerate(targetlist): + print(target['name'], + target['ami'], + "%s@%s"%(target['user'], instances[ii].public_ip_address)) + +# kill any connections +fabric.network.disconnect_all() diff --git a/tests/letstest/scripts/boulder_config.sh b/tests/letstest/scripts/boulder_config.sh new file mode 100755 index 000000000..1ef63ca10 --- /dev/null +++ b/tests/letstest/scripts/boulder_config.sh @@ -0,0 +1,32 @@ +#!/bin/bash -x + +# Configures and Launches Boulder Server installed on +# us-east-1 ami-5f490b35 bouldertestserver (boulder commit 8b433f54dab) + +# fetch instance data from EC2 metadata service +public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) +public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) +private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) + +# get local DNS resolver for VPC +resolver_ip=$(grep nameserver /etc/resolv.conf |cut -d" " -f2 |head -1) +resolver=$resolver_ip':53' + +# modifies integration testing boulder setup for local AWS VPC network +# connections instead of localhost +cd $GOPATH/src/github.com/letsencrypt/boulder +# configure boulder to receive outside connection on 4000 +sed -i '/listenAddress/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json +sed -i '/baseURL/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json +# change test ports to real +sed -i '/httpPort/ s/5002/80/' ./test/boulder-config.json +sed -i '/httpsPort/ s/5001/443/' ./test/boulder-config.json +sed -i '/tlsPort/ s/5001/443/' ./test/boulder-config.json +# set local dns resolver +sed -i '/dnsResolver/ s/127.0.0.1:8053/'$resolver'/' ./test/boulder-config.json + +# start rabbitMQ +#go run cmd/rabbitmq-setup/main.go -server amqp://localhost +# start acme services +#nohup ./start.py >& /dev/null < /dev/null & +#./start.py diff --git a/tests/letstest/scripts/boulder_install.sh b/tests/letstest/scripts/boulder_install.sh new file mode 100755 index 000000000..b5ddf2c5b --- /dev/null +++ b/tests/letstest/scripts/boulder_install.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x + +# >>>> only tested on Ubuntu 14.04LTS <<<< + +# non-interactive install of mariadb and other dependencies +export DEBIAN_FRONTEND=noninteractive +sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS' +sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS' +apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server +sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');" + +# install go +wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz +tar xzvf go1.5.1.linux-amd64.tar.gz +mkdir gocode +echo "export GOROOT=/home/ubuntu/go \n\ + export GOPATH=/home/ubuntu/gocode\n\ + export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc + +# install boulder and its go dependencies +go get -d github.com/letsencrypt/boulder/... +cd $GOPATH/src/github.com/letsencrypt/boulder +wget https://github.com/jsha/boulder-tools/raw/master/goose.gz +mkdir $GOPATH/bin +zcat goose.gz > $GOPATH/bin/goose +chmod +x $GOPATH/bin/goose +./test/create_db.sh +go get github.com/jsha/listenbuddy diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh new file mode 100755 index 000000000..a048a1ad0 --- /dev/null +++ b/tests/letstest/scripts/test_apache2.sh @@ -0,0 +1,60 @@ +#!/bin/bash -x + +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution + +if [ "$OS_TYPE" = "ubuntu" ] +then + CONFFILE=/etc/apache2/sites-available/000-default.conf + sudo apt-get update + sudo apt-get -y --no-upgrade install apache2 #curl + # For apache 2.4, set up ServerName + sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE + sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE +elif [ "$OS_TYPE" = "centos" ] +then + CONFFILE=/etc/httpd/conf/httpd.conf + sudo setenforce 0 || true #disable selinux + sudo yum -y install httpd + sudo service httpd start + sudo mkdir -p /var/www/$PUBLIC_HOSTNAME/public_html + sudo chmod -R oug+rwx /var/www + sudo chmod -R oug+rw /etc/httpd + sudo echo 'foobar' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html + sudo mkdir /etc/httpd/sites-available #letsencrypt requires this... + sudo mkdir /etc/httpd/sites-enabled #letsencrypt requires this... + #sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf + sudo echo """ + + ServerName $PUBLIC_HOSTNAME + DocumentRoot /var/www/$PUBLIC_HOSTNAME/public_html + ErrorLog /var/www/$PUBLIC_HOSTNAME/error.log + CustomLog /var/www/$PUBLIC_HOSTNAME/requests.log combined +""" >> /etc/httpd/conf.d/$PUBLIC_HOSTNAME.conf + #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/ +fi + +# run letsencrypt-apache2 via letsencrypt-auto +cd letsencrypt +./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \ + --renew-by-default --redirect --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +if [ "$OS_TYPE" = "ubuntu" ] ; then + export LETSENCRYPT="$HOME/.local/share/letsencrypt/bin/letsencrypt" + tests/apache-conf-files/hackish-apache-test --debian-modules +else + echo Not running hackish apache tests on $OS_TYPE +fi + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +# return error if any of the subtests failed +if [ "$FAIL" = 1 ] ; then + return 1 +fi diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh new file mode 100755 index 000000000..70f8a2293 --- /dev/null +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -0,0 +1,18 @@ +#!/bin/bash -xe + +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution + +cd letsencrypt +#git checkout v0.1.0 use --branch instead +SAVE="$PIP_EXTRA_INDEX_URL" +unset PIP_EXTRA_INDEX_URL +./letsencrypt-auto -v --debug --version + +export PIP_EXTRA_INDEX_URL="$SAVE" + +if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then + echo upgrade appeared to fail + exit 1 +fi +echo upgrade appeared to be successful diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh new file mode 100755 index 000000000..10d7c3b5e --- /dev/null +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -0,0 +1,15 @@ +#!/bin/bash -x + +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution + +# with curl, instance metadata available from EC2 metadata service: +#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) +#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) +#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) + +cd letsencrypt +./letsencrypt-auto certonly -v --standalone --debug \ + --text --agree-dev-preview --agree-tos \ + --renew-by-default --redirect \ + --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh new file mode 100755 index 000000000..476ad8bde --- /dev/null +++ b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh @@ -0,0 +1,7 @@ +#!/bin/bash -x + +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution + +cd letsencrypt +# help installs virtualenv and does nothing else +./letsencrypt-auto -v --help all diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh new file mode 100755 index 000000000..f7f325d5c --- /dev/null +++ b/tests/letstest/scripts/test_tox.sh @@ -0,0 +1,80 @@ +#!/bin/bash -x +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="venv" +# The path to the letsencrypt-auto script. Everything that uses these might +# at some point be inlined... +LEA_PATH=./letsencrypt/ +VENV_PATH=${LEA_PATH/$VENV_NAME} +VENV_BIN=${VENV_PATH}/bin +BOOTSTRAP=${LEA_PATH}/bootstrap + +SUDO=sudo + +ExperimentalBootstrap() { + # Arguments: Platform name, boostrap script name, SUDO command (iff needed) + if [ "$2" != "" ] ; then + echo "Bootstrapping dependencies for $1..." + if [ "$3" != "" ] ; then + "$3" "$BOOTSTRAP/$2" + else + "$BOOTSTRAP/$2" + fi + fi +} + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -f $BOOTSTRAP/debian.sh ] ; then + echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" + exit 1 +fi + +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO $BOOTSTRAP/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO $BOOTSTRAP/_rpm_common.sh +elif `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + $SUDO $BOOTSTRAP/_suse_common.sh +elif [ -f /etc/arch-release ] ; then + if [ "$DEBUG" = 1 ] ; then + echo "Bootstrapping dependencies for Archlinux..." + $SUDO $BOOTSTRAP/archlinux.sh + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi +elif [ -f /etc/manjaro-release ] ; then + ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" +elif [ -f /etc/gentoo-release ] ; then + ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" +elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" +elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root +elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" +else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info" +fi +echo "Bootstrapped!" + +cd letsencrypt +./bootstrap/dev/venv.sh +PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + +if [ $PYVER -eq 26 ] ; then + venv/bin/tox -e py26 +else + venv/bin/tox -e py27 +fi diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml new file mode 100644 index 000000000..506225f86 --- /dev/null +++ b/tests/letstest/targets.yaml @@ -0,0 +1,99 @@ +targets: + #----------------------------------------------------------------------------- + #Ubuntu + - ami: ami-26d5af4c + name: ubuntu15.10 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-d92e6bb3 + name: ubuntu15.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-7b89cc11 + name: ubuntu14.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-9295d0f8 + name: ubuntu14.04LTS_32bit + type: ubuntu + virt: pv + user: ubuntu + - ami: ami-0611546c + name: ubuntu12.04LTS + type: ubuntu + virt: hvm + user: ubuntu + #----------------------------------------------------------------------------- + # Debian + - ami: ami-116d857a + name: debian8.1 + type: ubuntu + virt: hvm + user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + - ami: ami-e0efab88 + name: debian7.8.aws.1 + type: ubuntu + virt: hvm + user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + - ami: ami-e6eeaa8e + name: debian7.8.aws.1_32bit + type: ubuntu + virt: pv + user: admin + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + #----------------------------------------------------------------------------- + # Other Redhat Distros + - ami: ami-60b6c60a + name: amazonlinux-2015.09.1 + type: centos + virt: hvm + user: ec2-user + - ami: ami-0d4cfd66 + name: amazonlinux-2015.03.1 + type: centos + virt: hvm + user: ec2-user + - ami: ami-a8d369c0 + name: RHEL7 + type: centos + virt: hvm + user: ec2-user + - ami: ami-518bfb3b + name: fedora23 + type: centos + virt: hvm + user: fedora + #----------------------------------------------------------------------------- + # CentOS + # These Marketplace AMIs must, irritatingly, have their terms manually + # agreed to on the AWS marketplace site for any new AWS account using them... + - ami: ami-61bbf104 + name: centos7 + type: centos + virt: hvm + user: centos + # centos6 requires EPEL repo added + - ami: ami-57cd8732 + name: centos6 + type: centos + virt: hvm + user: centos + userdata: | + #cloud-config + runcmd: + - yum install -y epel-release + - iptables -F diff --git a/tools/release.sh b/tools/release.sh index eeabfd4a3..172f6fea1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -86,7 +86,7 @@ SetVersion() { done sed -i "s/^__version.*/__version__ = '$ver'/" letsencrypt/__init__.py - git add -p $SUBPKGS # interactive user input + git add -p letsencrypt $SUBPKGS # interactive user input } SetVersion "$version" git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" diff --git a/tox.ini b/tox.ini index d1fafe20f..1abe1cf39 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ envlist = py26,py27,py33,py34,py35,cover,lint commands = pip install -e acme[testing] nosetests -v acme - pip install -r py26reqs.txt -e .[testing] + pip install -e .[testing] nosetests -v letsencrypt pip install -e letsencrypt-apache nosetests -v letsencrypt_apache