diff --git a/.travis.yml b/.travis.yml index 67da27d00..1d2f7b1db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ env: global: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH + matrix: include: - python: "2.6" @@ -47,6 +48,10 @@ matrix: env: TOXENV=py34 - python: "3.5" env: TOXENV=py35 + - sudo: required + env: TOXENV=le_auto + services: docker + before_install: # Only build pushes to the master branch, PRs, and branches beginning with # `test-`. This reduces the number of simultaneous Travis runs, which speeds @@ -67,7 +72,7 @@ addons: apt: sources: - augeas - packages: # keep in sync with bootstrap/ubuntu.sh and Boulder + packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder. - python-dev - python-virtualenv - gcc diff --git a/Dockerfile b/Dockerfile index da0110604..71e217659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # directories in its path. -COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh -RUN /opt/letsencrypt/src/ubuntu.sh && \ +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/Dockerfile-dev b/Dockerfile-dev index 61908d470..e4a22bea7 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto -RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/Vagrantfile b/Vagrantfile index 3b9d4dc3a..678abdf72 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ VAGRANTFILE_API_VERSION = "2" $ubuntu_setup_script = < /dev/null 2>&1; then - virtualenv="virtualenv" -fi - -if apt-cache show python-virtualenv > /dev/null 2>&1; then - virtualenv="$virtualenv python-virtualenv" -fi - -augeas_pkg="libaugeas0 augeas-lenses" -AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` - -AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - if echo $BACKPORT_NAME | grep -q wheezy ; then - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' - fi - - echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list - apt-get update - fi - fi - apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - -} - - -if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Let's Encrypt apache plugin..." - fi - # XXX add a case for ubuntu PPAs -fi - -apt-get install -y --no-install-recommends \ - python \ - python-dev \ - $virtualenv \ - gcc \ - dialog \ - $augeas_pkg \ - libssl-dev \ - libffi-dev \ - ca-certificates \ - - - -if ! command -v virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting - exit 1 -fi diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh deleted file mode 100755 index f49dc00f0..000000000 --- a/bootstrap/_gentoo_common.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -PACKAGES=" - dev-lang/python:2.7 - dev-python/virtualenv - dev-util/dialog - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - -case "$PACKAGE_MANAGER" in - (paludis) - cave resolve --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - pmerge --noreplace $PACKAGES - ;; - (portage|*) - emerge --noreplace $PACKAGES - ;; -esac diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh deleted file mode 100755 index 73890155e..000000000 --- a/bootstrap/_rpm_common.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -# Tested with: -# - Fedora 22, 23 (x64) -# - Centos 7 (x64: on DigitalOcean droplet) -# - CentOS 7 Minimal install in a Hyper-V VM - -if type dnf 2>/dev/null -then - tool=dnf -elif type yum 2>/dev/null -then - tool=yum - -else - echo "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 -fi - -# Some distros and older versions of current distros use a "python27" -# instead of "python" naming convention. Try both conventions. -if ! $tool install -y \ - python \ - python-devel \ - python-virtualenv \ - python-tools \ - python-pip -then - if ! $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv \ - python27-tools \ - python27-pip - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi -fi - -if ! $tool install -y \ - gcc \ - dialog \ - augeas-libs \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates -then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 -fi - - -if $tool list installed "httpd" >/dev/null 2>&1; then - if ! $tool install -y mod_ssl - then - echo "Apache found, but mod_ssl could not be installed." - fi -fi diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh deleted file mode 100755 index efeebe4f8..000000000 --- a/bootstrap/_suse_common.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# SLE12 don't have python-virtualenv - -zypper -nq in -l \ - python \ - python-devel \ - python-virtualenv \ - gcc \ - dialog \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates \ diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/archlinux.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/centos.sh b/bootstrap/centos.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/centos.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/debian.sh b/bootstrap/debian.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/debian.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/dev/README b/bootstrap/dev/README deleted file mode 100644 index 759496187..000000000 --- a/bootstrap/dev/README +++ /dev/null @@ -1 +0,0 @@ -This directory contains developer setup. diff --git a/bootstrap/fedora.sh b/bootstrap/fedora.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/fedora.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh deleted file mode 100755 index 4482c35cd..000000000 --- a/bootstrap/freebsd.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -xe - -pkg install -Ay \ - python \ - py27-virtualenv \ - augeas \ - libffi \ diff --git a/bootstrap/gentoo.sh b/bootstrap/gentoo.sh deleted file mode 120000 index 125d6a592..000000000 --- a/bootstrap/gentoo.sh +++ /dev/null @@ -1 +0,0 @@ -_gentoo_common.sh \ No newline at end of file diff --git a/bootstrap/install-deps.sh b/bootstrap/install-deps.sh deleted file mode 100755 index e907e7035..000000000 --- a/bootstrap/install-deps.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -e -# -# Install OS dependencies. In the glorious future, letsencrypt-auto will -# source this... - -if test "`id -u`" -ne "0" ; then - SUDO=sudo -else - SUDO= -fi - -BOOTSTRAP=`dirname $0` -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/arch-release ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif [ -f /etc/gentoo-release ] ; then - echo "Bootstrapping dependencies for Gentoo-based OSes..." - $SUDO $BOOTSTRAP/_gentoo_common.sh -elif uname | grep -iq FreeBSD ; then - echo "Bootstrapping dependencies for FreeBSD..." - $SUDO $BOOTSTRAP/freebsd.sh -elif `grep -qs openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE.." - $SUDO $BOOTSTRAP/suse.sh -elif uname | grep -iq Darwin ; then - echo "Bootstrapping dependencies for Mac OS X..." - echo "WARNING: Mac support is very experimental at present..." - $BOOTSTRAP/mac.sh -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - exit 1 -fi diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh deleted file mode 100755 index 4d1fb8208..000000000 --- a/bootstrap/mac.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e -if ! hash brew 2>/dev/null; then - echo "Homebrew Not Installed\nDownloading..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -fi - -brew install augeas -brew install dialog - -if ! hash pip 2>/dev/null; then - echo "pip Not Installed\nInstalling python from Homebrew..." - brew install python -fi - -if ! hash virtualenv 2>/dev/null; then - echo "virtualenv Not Installed\nInstalling with pip" - pip install virtualenv -fi diff --git a/bootstrap/manjaro.sh b/bootstrap/manjaro.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/manjaro.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/suse.sh b/bootstrap/suse.sh deleted file mode 120000 index fc4c1dee4..000000000 --- a/bootstrap/suse.sh +++ /dev/null @@ -1 +0,0 @@ -_suse_common.sh \ No newline at end of file diff --git a/bootstrap/ubuntu.sh b/bootstrap/ubuntu.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/ubuntu.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh deleted file mode 100755 index 5042178d9..000000000 --- a/bootstrap/venv.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -e -# -# Installs and updates letencrypt virtualenv -# -# USAGE: source ./dev/venv.sh - - -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -VENV_NAME="letsencrypt" -VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - virtualenv --no-site-packages --python ${LE_PYTHON:-python2} $VENV_PATH -fi - -. $VENV_PATH/bin/activate -pip install -U setuptools -pip install -U pip - -pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx - -echo -echo "Congratulations, Let's Encrypt has been successfully installed/updated!" -echo -printf "%s" "Your prompt should now be prepended with ($VENV_NAME). Next " -printf "time, if the prompt is different, 'source' this script again " -printf "before running 'letsencrypt'." -echo -echo -echo "You can now run 'letsencrypt --help'." diff --git a/docs/api/renewer.rst b/docs/api/renewer.rst deleted file mode 100644 index cc42c6eab..000000000 --- a/docs/api/renewer.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`letsencrypt.renewer` --------------------------- - -.. automodule:: letsencrypt.renewer - :members: diff --git a/docs/ciphers.rst b/docs/ciphers.rst index fb854f307..c8ff26117 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -107,7 +107,7 @@ and the version implemented by the Let's Encrypt client will be the version that was most current as of the release date of each client version. Mozilla offers three separate sets of cryptographic options, which trade off security and compatibility differently. These are -referred to as as the "Modern", "Intermediate", and "Old" configurations +referred to as the "Modern", "Intermediate", and "Old" configurations (in order from most secure to least secure, and least-backwards compatible to most-backwards compatible). The client will follow the Mozilla defaults for the *Intermediate* configuration by default, at least with regards to diff --git a/docs/conf.py b/docs/conf.py index 21bcc6817..739d6ee43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -281,8 +281,6 @@ man_pages = [ [project], 7), ('man/letsencrypt', 'letsencrypt', u'letsencrypt script documentation', [project], 1), - ('man/letsencrypt-renewer', 'letsencrypt-renewer', - u'letsencrypt-renewer script documentation', [project], 1), ] # If true, show URL addresses after external links. diff --git a/docs/contributing.rst b/docs/contributing.rst index 329f92461..8a27d565e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -23,7 +23,7 @@ once: git clone https://github.com/letsencrypt/letsencrypt cd letsencrypt ./letsencrypt-auto-source/letsencrypt-auto --os-packages-only - ./bootstrap/dev/venv.sh + ./tools/venv.sh Then in each shell where you're working on the client, do: @@ -300,7 +300,7 @@ Steps: 1. Write your code! 2. Make sure your environment is set up properly and that you're in your - virtualenv. You can do this by running ``./bootstrap/dev/venv.sh``. + virtualenv. You can do this by running ``./tools/venv.sh``. (this is a **very important** step) 3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. Fix any errors. diff --git a/docs/man/letsencrypt-renewer.rst b/docs/man/letsencrypt-renewer.rst deleted file mode 100644 index 8fd232fa8..000000000 --- a/docs/man/letsencrypt-renewer.rst +++ /dev/null @@ -1 +0,0 @@ -.. program-output:: letsencrypt-renewer --help diff --git a/docs/using.rst b/docs/using.rst index eb7c3962e..9ee16dffd 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -16,27 +16,12 @@ letsencrypt-auto ---------------- ``letsencrypt-auto`` is a wrapper which installs some dependencies -from your OS standard package repositories (e.g using `apt-get` or +from your OS standard package repositories (e.g. using `apt-get` or `yum`), and for other dependencies it sets up a virtualized Python environment with packages downloaded from PyPI [#venv]_. It also provides automated updates. -Firstly, please `install Git`_ and run the following commands: - -.. code-block:: shell - - git clone https://github.com/letsencrypt/letsencrypt - cd letsencrypt - - -.. _`install Git`: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git - -.. note:: On RedHat/CentOS 6 you will need to enable the EPEL_ - repository before install. - -.. _EPEL: http://fedoraproject.org/wiki/EPEL - -To install and run the client you just need to type: +To install and run the client, just type... .. code-block:: shell @@ -182,18 +167,19 @@ interested, you can also :ref:`write your own plugin `. Renewal ======= -.. note:: Let's Encrypt CA issues short lived certificates (90 +.. note:: Let's Encrypt CA issues short-lived certificates (90 days). Make sure you renew the certificates at least once in 3 months. In order to renew certificates simply call the ``letsencrypt`` (or -letsencrypt-auto_) again, and use the same values when prompted. You -can automate it slightly by passing necessary flags on the CLI (see -`--help all`), or even further using the :ref:`config-file`. The -``--renew-by-default`` flag may be helpful for automating renewal. If -you're sure that UI doesn't prompt for any details you can add the -command to ``crontab`` (make it less than every 90 days to avoid -problems, say every month). +letsencrypt-auto_) again, and use the same values when prompted. You can +automate it slightly by passing necessary flags on the CLI (see `--help +all`), or even further using the :ref:`config-file`. The ``--force-renew`` +flag may be helpful for automating renewal; it causes the expiration time +of the certificate(s) to be ignored when considering renewal. If you're +sure that UI doesn't prompt for any details you can add the command to +``crontab`` (make it less than every 90 days to avoid problems, say +every month). Please note that the CA will send notification emails to the address you provide if you do not renew certificates that are about to expire. diff --git a/letsencrypt-apache/letsencrypt_apache/centos-options-ssl-apache.conf b/letsencrypt-apache/letsencrypt_apache/centos-options-ssl-apache.conf new file mode 100644 index 000000000..fbe8da0f2 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/centos-options-ssl-apache.conf @@ -0,0 +1,21 @@ +# Baseline setting to Include for SSL sites + +SSLEngine on + +# Intermediate configuration, tweak to your needs +SSLProtocol all -SSLv2 -SSLv3 +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLHonorCipherOrder on + +SSLOptions +StrictRequire + +# Add vhost name to log entries: +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined +LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common + +#CustomLog /var/log/apache2/access.log vhost_combined +#LogLevel warn +#ErrorLog /var/log/apache2/error.log + +# Always ensure Cookies have "Secure" set (JAH 2012/1) +#Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4" diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 8818923f4..0ab16ff06 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -305,6 +305,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not vhost.ssl: vhost = self.make_vhost_ssl(vhost) + self._add_servername_alias(target_name, vhost) self.assoc[target_name] = vhost return vhost @@ -335,6 +336,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "VirtualHost not able to be selected.") + self._add_servername_alias(target_name, vhost) self.assoc[target_name] = vhost return vhost @@ -353,7 +355,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Points 1 - Address name with no SSL best_candidate = None best_points = 0 - for vhost in self.vhosts: if vhost.modmacro is True: continue @@ -643,11 +644,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if self.conf("handle-modules"): - if "ssl_module" not in self.parser.modules: - self.enable_mod("ssl", temp=temp) if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) + if "ssl_module" not in self.parser.modules: + self.enable_mod("ssl", temp=temp) def make_addrs_sni_ready(self, addrs): """Checks to see if the server is ready for SNI challenges. @@ -692,7 +693,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Reload augeas to take into account the new vhost self.aug.load() - # Get Vhost augeas path for new vhost vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % (ssl_fp, parser.case_i("VirtualHost"))) @@ -709,6 +709,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Add directives self._add_dummy_ssl_directives(vh_p) + self.save() # Log actions and create save notes logger.info("Created an SSL vhost at %s", ssl_fp) @@ -859,6 +860,22 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_key_file_path") self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + def _add_servername_alias(self, target_name, vhost): + fp = vhost.filep + vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % + (fp, parser.case_i("VirtualHost"))) + if not vh_p: + return + vh_path = vh_p[0] + if (self.parser.find_dir("ServerName", target_name, start=vh_path, exclude=False) + or self.parser.find_dir("ServerAlias", target_name, start=vh_path, exclude=False)): + return + if not self.parser.find_dir("ServerName", None, start=vh_path, exclude=False): + self.parser.add_dir(vh_path, "ServerName", target_name) + else: + self.parser.add_dir(vh_path, "ServerAlias", target_name) + self._add_servernames(vhost) + def _add_name_vhost_if_necessary(self, vhost): """Add NameVirtualHost Directives if necessary for new vhost. @@ -874,9 +891,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # See if the exact address appears in any other vhost # Remember 1.1.1.1:* == 1.1.1.1 -> hence any() for addr in vhost.addrs: + # In Apache 2.2, when a NameVirtualHost directive is not + # set, "*" and "_default_" will conflict when sharing a port + if addr.get_addr() in ("*", "_default_"): + addrs = [obj.Addr((a, addr.get_port(),)) + for a in ("*", "_default_")] + for test_vh in self.vhosts: if (vhost.filep != test_vh.filep and - any(test_addr == addr for + any(test_addr in addrs for test_addr in test_vh.addrs) and not self.is_name_vhost(addr)): self.add_name_vhost(addr) @@ -1587,4 +1610,4 @@ def install_ssl_options_conf(options_ssl): # Check to make sure options-ssl.conf is installed if not os.path.isfile(options_ssl): - shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl) + shutil.copyfile(constants.os_constant("MOD_SSL_CONF_SRC"), options_ssl) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index fe5ef3335..50156444b 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -16,7 +16,9 @@ CLI_DEFAULTS_DEBIAN = dict( le_vhost_ext="-le-ssl.conf", handle_mods=True, handle_sites=True, - challenge_location="/etc/apache2" + challenge_location="/etc/apache2", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "options-ssl-apache.conf") ) CLI_DEFAULTS_CENTOS = dict( server_root="/etc/httpd", @@ -31,7 +33,9 @@ CLI_DEFAULTS_CENTOS = dict( le_vhost_ext="-le-ssl.conf", handle_mods=False, handle_sites=False, - challenge_location="/etc/httpd/conf.d" + challenge_location="/etc/httpd/conf.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "centos-options-ssl-apache.conf") ) CLI_DEFAULTS_GENTOO = dict( server_root="/etc/apache2", @@ -46,7 +50,9 @@ CLI_DEFAULTS_GENTOO = dict( le_vhost_ext="-le-ssl.conf", handle_mods=False, handle_sites=False, - challenge_location="/etc/apache2/vhosts.d" + challenge_location="/etc/apache2/vhosts.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "options-ssl-apache.conf") ) CLI_DEFAULTS = { "debian": CLI_DEFAULTS_DEBIAN, @@ -62,11 +68,6 @@ CLI_DEFAULTS = { MOD_SSL_CONF_DEST = "options-ssl-apache.conf" """Name of the mod_ssl config file as saved in `IConfig.config_dir`.""" -MOD_SSL_CONF_SRC = pkg_resources.resource_filename( - "letsencrypt_apache", "options-ssl-apache.conf") -"""Path to the Apache mod_ssl config file found in the Let's Encrypt -distribution.""" - AUGEAS_LENS_DIR = pkg_resources.resource_filename( "letsencrypt_apache", "augeas_lens") """Path to the Augeas lens directory""" diff --git a/letsencrypt-apache/letsencrypt_apache/display_ops.py b/letsencrypt-apache/letsencrypt_apache/display_ops.py index d05a995ba..6a2308d73 100644 --- a/letsencrypt-apache/letsencrypt_apache/display_ops.py +++ b/letsencrypt-apache/letsencrypt_apache/display_ops.py @@ -4,6 +4,7 @@ import os import zope.component +from letsencrypt import errors from letsencrypt import interfaces import letsencrypt.display.util as display_util @@ -78,12 +79,18 @@ def _vhost_menu(domain, vhosts): name_size=disp_name_size) ) - code, tag = zope.component.getUtility(interfaces.IDisplay).menu( - "We were unable to find a vhost with a ServerName " - "or Address of {0}.{1}Which virtual host would you " - "like to choose?".format( - domain, os.linesep), - choices, help_label="More Info", ok_label="Select") + try: + code, tag = zope.component.getUtility(interfaces.IDisplay).menu( + "We were unable to find a vhost with a ServerName " + "or Address of {0}.{1}Which virtual host would you " + "like to choose?".format(domain, os.linesep), + choices, help_label="More Info", ok_label="Select") + except errors.MissingCommandlineFlag as e: + msg = ("Failed to run Apache plugin non-interactively{1}{0}{1}" + "(The best solution is to add ServerName or ServerAlias " + "entries to the VirtualHost directives of your apache " + "configuration files.)".format(e, os.linesep)) + raise errors.MissingCommandlineFlag(msg) return code, tag diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index cc7f2ec42..3c13aae5f 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -597,7 +597,7 @@ class ApacheParser(object): .. todo:: Make sure that files are included """ - default = self._set_user_config_file() + default = self.loc["root"] temp = os.path.join(self.root, "ports.conf") if os.path.isfile(temp): @@ -618,23 +618,6 @@ class ApacheParser(object): raise errors.NoInstallationError("Could not find configuration root") - def _set_user_config_file(self): - """Set the appropriate user configuration file - - .. todo:: This will have to be updated for other distros versions - - :param str root: pathname which contains the user config - - """ - # Basic check to see if httpd.conf exists and - # in hierarchy via direct include - # httpd.conf was very common as a user file in Apache 2.2 - if (os.path.isfile(os.path.join(self.root, "httpd.conf")) and - self.find_dir("Include", "httpd.conf", self.loc["root"])): - return os.path.join(self.root, "httpd.conf") - else: - return os.path.join(self.root, "apache2.conf") - def case_i(string): """Returns case insensitive regex. diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 00a98e33a..5b15a20d1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -161,6 +161,7 @@ class TwoVhost80Test(util.ApacheTest): def test_choose_vhost_select_vhost_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[0] chosen_vhost = self.config.choose_vhost("none.com") + self.vh_truth[0].aliases.add("none.com") self.assertEqual( self.vh_truth[0].get_names(), chosen_vhost.get_names()) @@ -192,8 +193,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual( self.vh_truth[0], self.config._find_best_vhost("encryption-example.demo")) - self.assertTrue( - self.config._find_best_vhost("does-not-exist.com") is None) + self.assertEqual( + self.config._find_best_vhost("does-not-exist.com"), None) def test_find_best_vhost_variety(self): # pylint: disable=protected-access @@ -428,9 +429,15 @@ class TwoVhost80Test(util.ApacheTest): self.config.parser.add_dir_to_ifmodssl = mock_add_dir self.config.prepare_server_https("443") + # Changing the order these modules are enabled breaks the reverter + self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb") + self.assertEqual(mock_enable.call_args[0][0], "ssl") self.assertEqual(mock_enable.call_args[1], {"temp": False}) self.config.prepare_server_https("8080", temp=True) + # Changing the order these modules are enabled breaks the reverter + self.assertEqual(mock_enable.call_args_list[2][0][0], "socache_shmcb") + self.assertEqual(mock_enable.call_args[0][0], "ssl") # Enable mod is temporary self.assertEqual(mock_enable.call_args[1], {"temp": True}) @@ -606,6 +613,14 @@ class TwoVhost80Test(util.ApacheTest): self.config._add_name_vhost_if_necessary(self.vh_truth[0]) self.assertTrue(self.config.save.called) + new_addrs = set() + for addr in self.vh_truth[0].addrs: + new_addrs.add(obj.Addr(("_default_", addr.get_port(),))) + + self.vh_truth[0].addrs = new_addrs + self.config._add_name_vhost_if_necessary(self.vh_truth[0]) + self.assertEqual(self.config.save.call_count, 2) + @mock.patch("letsencrypt_apache.configurator.tls_sni_01.ApacheTlsSni01.perform") @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") def test_perform(self, mock_restart, mock_perform): diff --git a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py index 6db319d87..f7fbac947 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py @@ -6,6 +6,7 @@ import mock import zope.component from letsencrypt.display import util as display_util +from letsencrypt import errors from letsencrypt_apache import obj @@ -31,6 +32,14 @@ class SelectVhostTest(unittest.TestCase): mock_util().menu.return_value = (display_util.OK, 3) self.assertEqual(self.vhosts[3], self._call(self.vhosts)) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") + def test_noninteractive(self, mock_util): + mock_util().menu.side_effect = errors.MissingCommandlineFlag("no vhost default") + try: + self._call(self.vhosts) + except errors.MissingCommandlineFlag as e: + self.assertTrue("VirtualHost directives" in e.message) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") def test_more_info_cancel(self, mock_util): mock_util().menu.side_effect = [ diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index e976bc9f6..2e6481aba 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -106,7 +106,7 @@ class BasicParserTest(util.ParserTest): def test_set_locations(self): with mock.patch("letsencrypt_apache.parser.os.path") as mock_path: - mock_path.isfile.side_effect = [True, False, False] + mock_path.isfile.side_effect = [False, False] # pylint: disable=protected-access results = self.parser._set_locations() @@ -114,16 +114,6 @@ class BasicParserTest(util.ParserTest): self.assertEqual(results["default"], results["listen"]) self.assertEqual(results["default"], results["name"]) - def test_set_user_config_file(self): - # pylint: disable=protected-access - path = os.path.join(self.parser.root, "httpd.conf") - open(path, 'w').close() - self.parser.add_dir(self.parser.loc["default"], "Include", - "httpd.conf") - - self.assertEqual( - path, self.parser._set_user_config_file()) - @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_variables(self, mock_cfg): mock_cfg.return_value = ( diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index fb86d2320..97a11e851 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -33,7 +33,7 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods pkg="letsencrypt_apache.tests") self.ssl_options = common.setup_ssl_options( - self.config_dir, constants.MOD_SSL_CONF_SRC, + self.config_dir, constants.os_constant("MOD_SSL_CONF_SRC"), constants.MOD_SSL_CONF_DEST) self.config_path = os.path.join(self.temp_dir, config_root) @@ -150,7 +150,7 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, ("default-ssl-port-only.conf/" "IfModule/VirtualHost")), - set([obj.Addr.fromstring("_default_:443")]), True, False), + set([obj.Addr.fromstring("_default_:443")]), True, False) ] return vh_truth diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index cbac3e0b2..a6553d890 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.1.dev0' +version = '0.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-auto b/letsencrypt-auto deleted file mode 100755 index 20465dbb1..000000000 --- a/letsencrypt-auto +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/sh -e -# -# A script to run the latest release version of the Let's Encrypt in a -# virtual environment -# -# Installs and updates the letencrypt virtualenv, and runs letsencrypt -# using that virtual environment. This allows the client to function decently -# without requiring specific versions of its dependencies from the operating -# system. - -# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, -# if you want to change where the virtual environment will be installed -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -VENV_NAME="letsencrypt" -VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} -VENV_BIN=${VENV_PATH}/bin -# The path to the letsencrypt-auto script. Everything that uses these might -# at some point be inlined... -LEA_PATH=`dirname "$0"` -BOOTSTRAP=${LEA_PATH}/bootstrap - -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) -for arg in "$@" ; do - # This first clause is redundant with the third, but hedging on portability - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then - VERBOSE=1 - elif [ "$arg" = "--debug" ] ; then - DEBUG=1 - fi -done - -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$DEBUG" = 1 ] ; then - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi - fi - else - echo "WARNING: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - exit 1 - fi -} - -DeterminePythonVersion() { - if command -v python2.7 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2.7} - elif command -v python27 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python27} - elif command -v python2 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2} - elif command -v python > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python} - else - echo "Cannot find any Pythons... please install one!" - exit 1 - fi - - PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -eq 26 ] ; then - ExperimentalBootstrap "Python 2.6" - elif [ $PYVER -lt 26 ] ; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." - exit 1 - fi -} - - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 - fi - - if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh - elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh - elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh - elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." - exit 1 - fi - elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" - elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" - elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" - elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root - elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" - else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - fi - - DeterminePythonVersion - echo "Creating virtual environment..." - if [ "$VERBOSE" = 1 ] ; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH - else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null - fi -else - DeterminePythonVersion -fi - - -printf "Updating letsencrypt and virtual environment dependencies..." -if [ "$VERBOSE" = 1 ] ; then - echo - $VENV_BIN/pip install -U setuptools - $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache - # nginx is buggy / disabled for now, but upgrade it if the user has - # installed it manually - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - $VENV_BIN/pip install -U letsencrypt letsencrypt-nginx - fi -else - $VENV_BIN/pip install -U setuptools > /dev/null - printf . - $VENV_BIN/pip install -U pip > /dev/null - printf . - # nginx is buggy / disabled for now... - $VENV_BIN/pip install -U letsencrypt > /dev/null - printf . - $VENV_BIN/pip install -U letsencrypt-apache > /dev/null - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - printf . - $VENV_BIN/pip install -U letsencrypt-nginx > /dev/null - fi - echo -fi - -# Explain what's about to happen, for the benefit of those getting sudo -# password prompts... -echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" -$SUDO $VENV_BIN/letsencrypt "$@" diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 120000 index 000000000..af7e83a70 --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1 @@ +letsencrypt-auto-source/letsencrypt-auto \ No newline at end of file diff --git a/letsencrypt-auto-source/Dockerfile b/letsencrypt-auto-source/Dockerfile index 667acfe5a..fd7fe4851 100644 --- a/letsencrypt-auto-source/Dockerfile +++ b/letsencrypt-auto-source/Dockerfile @@ -7,7 +7,6 @@ FROM ubuntu:trusty RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea # Let that user sudo: -RUN adduser lea sudo RUN sed -i.bkp -e \ 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \ /etc/sudoers diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9eebddc9d..24b62e342 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.2.1.dev0" +LE_AUTO_VERSION="0.4.0.dev0" # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support @@ -105,9 +105,7 @@ DeterminePythonVersion() { fi PYVER=`"$LE_PYTHON" --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -eq 26 ]; then - ExperimentalBootstrap "Python 2.6" - elif [ $PYVER -lt 26 ]; then + if [ $PYVER -lt 26 ]; then echo "You have an ancient version of Python entombed in your operating system..." echo "This isn't going to work; you'll need at least version 2.6." exit 1 @@ -292,7 +290,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./bootstrap/dev/_common_venv.sh + # ./tools/_venv_common.sh deps=" python2 @@ -374,7 +372,7 @@ Bootstrap() { elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon - elif `grep -q openSUSE /etc/os-release` ; then + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then echo "Bootstrapping dependencies for openSUSE-based OSes..." BootstrapSuseCommon elif [ -f /etc/arch-release ]; then @@ -438,8 +436,8 @@ if [ "$NO_SELF_UPGRADE" = 1 ]; then # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" # This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install -r -e acme -e . -e letsencrypt-apache`, `pip freeze`, -# and then gather the hashes. +# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# then use `hashin` or a more secure method to gather the hashes. # sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ # sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ @@ -508,9 +506,16 @@ idna==2.0 # sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA ipaddress==1.0.16 +# sha256: 54vpwKDfy6xxL-BPv5K5bN2ugLG4QvJCSCFMhJbwBu8 +# sha256: Syb_TnEQ23butvWntkqCYjg51ZXCA47tpmLyott46Xw +linecache2==1.0.0 + # sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ ndg-httpsclient==0.4.0 +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs parsedatetime==1.5 @@ -596,6 +601,14 @@ requests==2.9.1 # sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo six==1.10.0 +# sha256: glPOvsSxkJTWfMXtWvmb8duhKFKSIm6Yoxkp-HpdayM +# sha256: BazGegmYDC7P7dNCP3rgEEg57MtV_GRXc-HKoJUcMDA +traceback2==1.4.0 + +# sha256: E_d9CHXbbZtDXh1PQedK1MwutuHVyCSZYJKzQw8Ii7g +# sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk +unittest2==1.1.0 + # sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms # sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI Werkzeug==0.11.3 @@ -625,17 +638,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: fYwCUXn3Wd_tKYHuPfufzDQZuDNng0HZb_th3xepC7U -# sha256: dKkCf9CZnKaDTFOof-KoazoewjKTSAVxZUJmnj_3i_U -acme==0.2.0 +# sha256: QMIkIvGF3mcJhGLAKRX7n5EVIPjOrfLtklN6ePjbJes +# sha256: fNFWiij6VxfG5o7u3oNbtrYKQ4q9vhzOLATfxNlozvQ +acme==0.3.0 -# sha256: 4x7K5lzKwm_GjYMojvUh053qL4EfIC5hGFmW370-7jI -# sha256: kcm3VmxXIGNS7ShcKFnYdA9AfXnqcbV_otMsADr1p2A -letsencrypt==0.2.0 +# sha256: qdnzpoRf_44QXKoktNoAKs2RBAxUta2Sr6GS0t_tAKo +# sha256: ELWJaHNvBZIqVPJYkla8yXLtXIuamqAf6f_VAFv16Uk +letsencrypt==0.3.0 -# sha256: AKuIT6b7gXXD2Cs7Qoem8ZrxcqBjABz1IgxhHGxmwX0 -# sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs -letsencrypt-apache==0.2.0 +# sha256: EypLpEw3-Tr8unw4aSFsHXgRiU8ZYLrJKOJohP2tC9M +# sha256: HYvP13GzA-DDJYwlfOoaraJO0zuYO48TCSAyTUAGCqA +letsencrypt-apache==0.3.0 # sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 # sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 7db9da58e..4bb5f97ea 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 8118a5f69..ad8c97a7f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -105,9 +105,7 @@ DeterminePythonVersion() { fi PYVER=`"$LE_PYTHON" --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -eq 26 ]; then - ExperimentalBootstrap "Python 2.6" - elif [ $PYVER -lt 26 ]; then + if [ $PYVER -lt 26 ]; then echo "You have an ancient version of Python entombed in your operating system..." echo "This isn't going to work; you'll need at least version 2.6." exit 1 @@ -130,7 +128,7 @@ Bootstrap() { elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon - elif `grep -q openSUSE /etc/os-release` ; then + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then echo "Bootstrapping dependencies for openSUSE-based OSes..." BootstrapSuseCommon elif [ -f /etc/arch-release ]; then diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index ad752634c..90a9d43c4 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -4,7 +4,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./bootstrap/dev/_common_venv.sh + # ./tools/_venv_common.sh deps=" python2 diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index bbda9f0b2..c83396de2 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -1,6 +1,6 @@ # This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install -r -e acme -e . -e letsencrypt-apache`, `pip freeze`, -# and then gather the hashes. +# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# then use `hashin` or a more secure method to gather the hashes. # sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ # sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ @@ -69,9 +69,16 @@ idna==2.0 # sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA ipaddress==1.0.16 +# sha256: 54vpwKDfy6xxL-BPv5K5bN2ugLG4QvJCSCFMhJbwBu8 +# sha256: Syb_TnEQ23butvWntkqCYjg51ZXCA47tpmLyott46Xw +linecache2==1.0.0 + # sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ ndg-httpsclient==0.4.0 +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs parsedatetime==1.5 @@ -157,6 +164,14 @@ requests==2.9.1 # sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo six==1.10.0 +# sha256: glPOvsSxkJTWfMXtWvmb8duhKFKSIm6Yoxkp-HpdayM +# sha256: BazGegmYDC7P7dNCP3rgEEg57MtV_GRXc-HKoJUcMDA +traceback2==1.4.0 + +# sha256: E_d9CHXbbZtDXh1PQedK1MwutuHVyCSZYJKzQw8Ii7g +# sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk +unittest2==1.1.0 + # sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms # sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI Werkzeug==0.11.3 @@ -186,17 +201,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: fYwCUXn3Wd_tKYHuPfufzDQZuDNng0HZb_th3xepC7U -# sha256: dKkCf9CZnKaDTFOof-KoazoewjKTSAVxZUJmnj_3i_U -acme==0.2.0 +# sha256: QMIkIvGF3mcJhGLAKRX7n5EVIPjOrfLtklN6ePjbJes +# sha256: fNFWiij6VxfG5o7u3oNbtrYKQ4q9vhzOLATfxNlozvQ +acme==0.3.0 -# sha256: 4x7K5lzKwm_GjYMojvUh053qL4EfIC5hGFmW370-7jI -# sha256: kcm3VmxXIGNS7ShcKFnYdA9AfXnqcbV_otMsADr1p2A -letsencrypt==0.2.0 +# sha256: qdnzpoRf_44QXKoktNoAKs2RBAxUta2Sr6GS0t_tAKo +# sha256: ELWJaHNvBZIqVPJYkla8yXLtXIuamqAf6f_VAFv16Uk +letsencrypt==0.3.0 -# sha256: AKuIT6b7gXXD2Cs7Qoem8ZrxcqBjABz1IgxhHGxmwX0 -# sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs -letsencrypt-apache==0.2.0 +# sha256: EypLpEw3-Tr8unw4aSFsHXgRiU8ZYLrJKOJohP2tC9M +# sha256: HYvP13GzA-DDJYwlfOoaraJO0zuYO48TCSAyTUAGCqA +letsencrypt-apache==0.3.0 # sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 # sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 4131fdcad..b7f448e83 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.1.dev0' +version = '0.4.0.dev0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index efa7e08b4..3d48d100f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -239,6 +239,7 @@ class NginxConfigurator(common.Plugin): def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. + The ranking gives preference to SSL vhosts. :param str target_name: The name to match :returns: list of dicts containing the vhost, the matching name, and @@ -309,10 +310,10 @@ class NginxConfigurator(common.Plugin): key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, le_key.pem) cert = acme_crypto_util.gen_ss_cert(key, domains=[socket.gethostname()]) - cert_path = os.path.join(tmp_dir, "cert.pem") cert_pem = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, cert) - with open(cert_path, 'w') as cert_file: + cert_file, cert_path = le_util.unique_file(os.path.join(tmp_dir, "cert.pem")) + with cert_file: cert_file.write(cert_pem) return cert_path, le_key.file diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index b5948b981..c1ff85185 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.1.dev0' +version = '0.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 00dbe4320..1dd7d7eba 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.2.1.dev0' +__version__ = '0.4.0.dev0' diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index c63d8c8d4..ffbd70ced 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -57,7 +57,7 @@ class AuthHandler(object): def get_authorizations(self, domains, best_effort=False): """Retrieve all authorizations for challenges. - :param set domains: Domains for authorization + :param list domains: Domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) @@ -480,6 +480,9 @@ def is_preferred(offered_challb, satisfied, return True +_ACME_PREFIX = "urn:acme:error:" + + _ERROR_HELP_COMMON = ( "To fix these errors, please make sure that your domain name was entered " "correctly and the DNS A record(s) for that domain contain(s) the " @@ -490,7 +493,9 @@ _ERROR_HELP = { "connection": _ERROR_HELP_COMMON + " Additionally, please check that your computer " "has a publicly routable IP address and that no firewalls are preventing " - "the server from communicating with the client.", + "the server from communicating with the client. If you're using the " + "webroot plugin, you should also verify that you are serving files " + "from the webroot path you provided.", "dnssec": _ERROR_HELP_COMMON + " Additionally, if you have DNSSEC enabled for " "your domain, please ensure that the signature is valid.", @@ -540,11 +545,13 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ + if typ.startswith(_ACME_PREFIX): + typ = typ[len(_ACME_PREFIX):] msg = ["The following errors were reported by the server:"] for achall in failed_achalls: msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % ( - achall.domain, achall.error.typ, achall.error.detail)) + achall.domain, typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index d2acb03bc..c98b3f0d7 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -6,7 +6,9 @@ from __future__ import print_function # (TODO: split this file into main.py and cli.py) import argparse import atexit +import copy import functools +import glob import json import logging import logging.handlers @@ -44,6 +46,28 @@ from letsencrypt.plugins import disco as plugins_disco logger = logging.getLogger(__name__) +# This is global scope in order to be able to extract type information from +# it later +_parser = None + +# These are the items which get pulled out of a renewal configuration +# file's renewalparams and actually used in the client configuration +# during the renewal process. We have to record their types here because +# the renewal configuration process loses this information. +STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", + "server", "account", "authenticator", "installer", + "standalone_supported_challenges"] +INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] + +# For help strings, figure out how the user ran us. +# When invoked from letsencrypt-auto, sys.argv[0] is something like: +# "/home/user/.local/share/letsencrypt/bin/letsencrypt" +# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before running +# letsencrypt-auto (and sudo stops us from seeing if they did), so it should only be used +# for purposes where inability to detect letsencrypt-auto fails safely + +fragment = os.path.join(".local", "share", "letsencrypt") +cli_command = "letsencrypt-auto" if fragment in sys.argv[0] else "letsencrypt" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want # to replace as much of it as we can... @@ -51,7 +75,7 @@ logger = logging.getLogger(__name__) # This is the stub to include in help generated by argparse SHORT_USAGE = """ - letsencrypt [SUBCOMMAND] [options] [-d domain] [-d domain] ... + {0} [SUBCOMMAND] [options] [-d domain] [-d domain] ... The Let's Encrypt agent can obtain and install HTTPS/TLS/SSL certificates. By default, it will attempt to use a webserver both for obtaining and installing @@ -60,12 +84,13 @@ the cert. Major SUBCOMMANDS are: (default) run Obtain & install a cert in your current webserver certonly Obtain cert, but do not install it (aka "auth") install Install a previously obtained cert in a server + renew Renew previously obtained certs that are near expiry revoke Revoke a previously obtained certificate rollback Rollback server configuration changes made during install config_changes Show changes made to server config during installation plugins Display information about installed plugins -""" +""".format(cli_command) # This is the short help for letsencrypt --help, where we disable argparse # altogether @@ -103,11 +128,15 @@ def usage_strings(plugins): return USAGE % (apache_doc, nginx_doc), SHORT_USAGE -def _find_domains(args, installer): - if not args.domains: +def _find_domains(config, installer): + if not config.domains: domains = display_ops.choose_names(installer) + # record in config.domains (so that it can be serialised in renewal config files), + # and set webroot_map entries if applicable + for d in domains: + _process_domain(config, d) else: - domains = args.domains + domains = config.domains if not domains: raise errors.Error("Please specify --domains, or --installer that " @@ -116,14 +145,14 @@ def _find_domains(args, installer): return domains -def _determine_account(args, config): +def _determine_account(config): """Determine which account to use. In order to make the renewer (configuration de/serialization) happy, - if ``args.account`` is ``None``, it will be updated based on the - user input. Same for ``args.email``. + if ``config.account`` is ``None``, it will be updated based on the + user input. Same for ``config.email``. - :param argparse.Namespace args: CLI arguments + :param argparse.Namespace config: CLI arguments :param letsencrypt.interface.IConfig config: Configuration object :param .AccountStorage account_storage: Account storage. @@ -136,8 +165,8 @@ def _determine_account(args, config): account_storage = account.AccountFileStorage(config) acme = None - if args.account is not None: - acc = account_storage.load(args.account) + if config.account is not None: + acc = account_storage.load(config.account) else: accounts = account_storage.find_all() if len(accounts) > 1: @@ -145,35 +174,37 @@ def _determine_account(args, config): elif len(accounts) == 1: acc = accounts[0] else: # no account registered yet - if args.email is None and not args.register_unsafely_without_email: - args.email = display_ops.get_email() + if config.email is None and not config.register_unsafely_without_email: + config.namespace.email = display_ops.get_email() def _tos_cb(regr): - if args.tos: + if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( regr.terms_of_service, config.server)) - return zope.component.getUtility(interfaces.IDisplay).yesno( - msg, "Agree", "Cancel") + obj = zope.component.getUtility(interfaces.IDisplay) + return obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) + except errors.MissingCommandlineFlag: + raise except errors.Error as error: logger.debug(error, exc_info=True) raise errors.Error( "Unable to register an account with ACME server") - args.account = acc.id + config.namespace.account = acc.id return acc, acme -def _init_le_client(args, config, authenticator, installer): +def _init_le_client(config, authenticator, installer): if authenticator is not None: # if authenticator was given, then we will need account... - acc, acme = _determine_account(args, config) + acc, acme = _determine_account(config) logger.debug("Picked account: %r", acc) # XXX #crypto_util.validate_key_csr(acc.key) @@ -193,15 +224,12 @@ def _find_duplicative_certs(config, domains): # Verify the directory is there le_util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) - for renewal_file in os.listdir(configs_dir): - if not renewal_file.endswith(".conf"): - continue + for renewal_file in _renewal_conf_files(cli_config): try: - full_path = os.path.join(configs_dir, renewal_file) - candidate_lineage = storage.RenewableCert(full_path, cli_config) + candidate_lineage = storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): - logger.warning("Renewal configuration file %s is broken. " - "Skipping.", full_path) + logger.warning("Renewal conf file %s is broken. Skipping.", renewal_file) + logger.debug("Traceback was:\n%s", traceback.format_exc()) continue # TODO: Handle these differently depending on whether they are # expired or still valid? @@ -220,7 +248,10 @@ def _find_duplicative_certs(config, domains): def _treat_as_renewal(config, domains): - """Determine whether there are duplicated names and how to handle them. + """Determine whether there are duplicated names and how to handle + them (renew, reinstall, newcert, or raising an error to stop + the client run if the user chooses to cancel the operation when + prompted). :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -247,8 +278,24 @@ def _treat_as_renewal(config, domains): elif subset_names_cert is not None: return _handle_subset_cert_request(config, domains, subset_names_cert) + +def _should_renew(config, lineage): + "Return true if any of the circumstances for automatic renewal apply." + if config.renew_by_default: + logger.info("Auto-renewal forced with --renew-by-default...") + return True + if lineage.should_autorenew(interactive=True): + logger.info("Cert is due for renewal, auto-renewing...") + return True + if config.dry_run: + logger.info("Cert not due for renewal, but simulating renewal for dry run") + return True + logger.info("Cert not yet due for renewal") + return False + + def _handle_identical_cert_request(config, cert): - """Figure out what to do if a cert has the same names as a perviously obtained one + """Figure out what to do if a cert has the same names as a previously obtained one :param storage.RenewableCert cert: @@ -256,17 +303,12 @@ def _handle_identical_cert_request(config, cert): :rtype: tuple """ - if config.renew_by_default: - logger.info("Auto-renewal forced with --force-renewal or --renew-by-default...") - return "renew", cert - if cert.should_autorenew(interactive=True): - logger.info("Cert is due for renewal, auto-renewing...") + if _should_renew(config, cert): return "renew", cert if config.reinstall: # Set with --reinstall, force an identical certificate to be # reinstalled without further prompting. return "reinstall", cert - question = ( "You have an existing certificate that contains exactly the same " "domains you requested and isn't close to expiry." @@ -282,7 +324,7 @@ def _handle_identical_cert_request(config, cert): "Cancel this operation and do nothing"] display = zope.component.getUtility(interfaces.IDisplay) - response = display.menu(question, choices, "OK", "Cancel") + response = display.menu(question, choices, "OK", "Cancel", default=0) if response[0] == "cancel" or response[1] == 2: # TODO: Add notification related to command-line options for # skipping the menu for this case. @@ -296,6 +338,7 @@ def _handle_identical_cert_request(config, cert): else: assert False, "This is impossible" + def _handle_subset_cert_request(config, domains, cert): """Figure out what to do if a previous cert had a subset of the names now requested @@ -317,7 +360,8 @@ def _handle_subset_cert_request(config, domains, cert): ", ".join(domains), br=os.linesep) if config.expand or config.renew_by_default or zope.component.getUtility( - interfaces.IDisplay).yesno(question, "Expand", "Cancel"): + interfaces.IDisplay).yesno(question, "Expand", "Cancel", + cli_flag="--expand (or in some cases, --duplicate)"): return "renew", cert else: reporter_util = zope.component.getUtility(interfaces.IReporter) @@ -362,16 +406,26 @@ def _report_new_cert(cert_path, fullchain_path): .format(and_chain, path, expiry)) reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY) -def _suggest_donate(): - "Suggest a donation to support Let's Encrypt" + +def _suggest_donation_if_appropriate(config): + """Potentially suggest a donation to support Let's Encrypt.""" + if not config.staging and not config.verb == "renew": # --dry-run implies --staging + reporter_util = zope.component.getUtility(interfaces.IReporter) + msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" + "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" + "Donating to EFF: https://eff.org/donate-le\n\n") + reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) + + + +def _report_successful_dry_run(config): reporter_util = zope.component.getUtility(interfaces.IReporter) - msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" - "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" - "Donating to EFF: https://eff.org/donate-le\n\n") - reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) + if config.verb != "renew": + reporter_util.add_message("The dry run was successful.", + reporter_util.HIGH_PRIORITY, on_crash=False) -def _auth_from_domains(le_client, config, domains): +def _auth_from_domains(le_client, config, domains, lineage=None): """Authenticate and enroll certificate.""" # Note: This can raise errors... caught above us though. This is now # a three-way case: reinstall (which results in a no-op here because @@ -380,7 +434,16 @@ def _auth_from_domains(le_client, config, domains): # (which results in treating the request as a renewal), or newcert # (which results in treating the request as a new certificate request). - action, lineage = _treat_as_renewal(config, domains) + # If lineage is specified, use that one instead of looking around for + # a matching one. + if lineage is None: + # This will find a relevant matching lineage that exists + action, lineage = _treat_as_renewal(config, domains) + else: + # Renewal, where we already know the specific lineage we're + # interested in + action = "renew" + if action == "reinstall": # The lineage already exists; allow the caller to try installing # it without getting a new certificate at all. @@ -392,25 +455,31 @@ def _auth_from_domains(le_client, config, domains): # https://github.com/letsencrypt/letsencrypt/pull/777/files#r40498574 new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) # TODO: Check whether it worked! <- or make sure errors are thrown (jdk) - lineage.save_successor( - lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), - new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) - - lineage.update_all_links_to(lineage.latest_common_version()) + if config.dry_run: + logger.info("Dry run: skipping updating lineage at %s", + os.path.dirname(lineage.cert)) + else: + lineage.save_successor( + lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), + new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain), + configuration.RenewerConfiguration(config.namespace)) + lineage.update_all_links_to(lineage.latest_common_version()) # TODO: Check return value of save_successor # TODO: Also update lineage renewal config with any relevant # configuration values from this attempt? <- Absolutely (jdkasten) elif action == "newcert": # TREAT AS NEW REQUEST lineage = le_client.obtain_and_enroll_certificate(domains) - if not lineage: + if lineage is False: raise errors.Error("Certificate could not be obtained") - _report_new_cert(lineage.cert, lineage.fullchain) + if not config.dry_run and not config.verb == "renew": + _report_new_cert(lineage.cert, lineage.fullchain) return lineage, action + def _avoid_invalidating_lineage(config, lineage, original_server): "Do not renew a valid cert with one from a staging server!" def _is_staging(srv): @@ -422,7 +491,7 @@ def _avoid_invalidating_lineage(config, lineage, original_server): open(lineage.cert).read()) # all our test certs are from happy hacker fake CA, though maybe one day # we should test more methodically - now_valid = not "fake" in repr(latest_cert.get_issuer()).lower() + now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if _is_staging(config.server): if not _is_staging(original_server) or now_valid: @@ -433,21 +502,6 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "a test certificate (domains: {0}). We will not do that " "unless you use the --break-my-certs flag!".format(names)) -def set_configurator(previously, now): - """ - Setting configurators multiple ways is okay, as long as they all agree - :param str previously: previously identified request for the installer/authenticator - :param str requested: the request currently being processed - """ - if now is None: - # we're not actually setting anything - return previously - if previously: - if previously != now: - msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}" - raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) - return now - def diagnose_configurator_problem(cfg_type, requested, plugins): """ @@ -481,39 +535,85 @@ def diagnose_configurator_problem(cfg_type, requested, plugins): raise errors.PluginSelectionError(msg) -def choose_configurator_plugins(args, config, plugins, verb): # pylint: disable=too-many-branches +def set_configurator(previously, now): """ - Figure out which configurator we're going to use - :raises error.PluginSelectionError if there was a problem + Setting configurators multiple ways is okay, as long as they all agree + :param str previously: previously identified request for the installer/authenticator + :param str requested: the request currently being processed + """ + if now is None: + # we're not actually setting anything + return previously + if previously: + if previously != now: + msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}" + raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) + return now + + +def cli_plugin_requests(config): + """ + Figure out which plugins the user requested with CLI and config options + + :returns: (requested authenticator string or None, requested installer string or None) + :rtype: tuple + """ + req_inst = req_auth = config.configurator + req_inst = set_configurator(req_inst, config.installer) + req_auth = set_configurator(req_auth, config.authenticator) + if config.nginx: + req_inst = set_configurator(req_inst, "nginx") + req_auth = set_configurator(req_auth, "nginx") + if config.apache: + req_inst = set_configurator(req_inst, "apache") + req_auth = set_configurator(req_auth, "apache") + if config.standalone: + req_auth = set_configurator(req_auth, "standalone") + if config.webroot: + req_auth = set_configurator(req_auth, "webroot") + if config.manual: + req_auth = set_configurator(req_auth, "manual") + logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) + return req_auth, req_inst + + +noninstaller_plugins = ["webroot", "manual", "standalone"] + + +def choose_configurator_plugins(config, plugins, verb): + """ + Figure out which configurator we're going to use, modifies + config.authenticator and config.istaller strings to reflect that choice if + necessary. + + :raises errors.PluginSelectionError if there was a problem + + :returns: (an `IAuthenticator` or None, an `IInstaller` or None) + :rtype: tuple """ + req_auth, req_inst = cli_plugin_requests(config) + # Which plugins do we need? - need_inst = need_auth = (verb == "run") + if verb == "run": + need_inst = need_auth = True + if req_auth in noninstaller_plugins and not req_inst: + msg = ('With the {0} plugin, you probably want to use the "certonly" command, eg:{1}' + '{1} {2} certonly --{0}{1}{1}' + '(Alternatively, add a --installer flag. See https://eff.org/letsencrypt-plugins' + '{1} and "--help plugins" for more information.)'.format( + req_auth, os.linesep, cli_command)) + + raise errors.MissingCommandlineFlag(msg) + else: + need_inst = need_auth = False if verb == "certonly": need_auth = True if verb == "install": need_inst = True - if args.authenticator: + if config.authenticator: logger.warn("Specifying an authenticator doesn't make sense in install mode") - # Which plugins did the user request? - req_inst = req_auth = args.configurator - req_inst = set_configurator(req_inst, args.installer) - req_auth = set_configurator(req_auth, args.authenticator) - if args.nginx: - req_inst = set_configurator(req_inst, "nginx") - req_auth = set_configurator(req_auth, "nginx") - if args.apache: - req_inst = set_configurator(req_inst, "apache") - req_auth = set_configurator(req_auth, "apache") - if args.standalone: - req_auth = set_configurator(req_auth, "standalone") - if args.webroot: - req_auth = set_configurator(req_auth, "webroot") - if args.manual: - req_auth = set_configurator(req_auth, "manual") - logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) - # Try to meet the user's request and/or ask them to pick plugins authenticator = installer = None if verb == "run" and req_auth == req_inst: @@ -545,18 +645,18 @@ def record_chosen_plugins(config, plugins, auth, inst): # TODO: Make run as close to auth + install as possible -# Possible difficulties: args.csr was hacked into auth -def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-locals +# Possible difficulties: config.csr was hacked into auth +def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals """Obtain a certificate and install.""" try: - installer, authenticator = choose_configurator_plugins(args, config, plugins, "run") - except errors.PluginSelectionError, e: + installer, authenticator = choose_configurator_plugins(config, plugins, "run") + except errors.PluginSelectionError as e: return e.message - domains = _find_domains(args, installer) + domains = _find_domains(config, installer) # TODO: Handle errors from _init_le_client? - le_client = _init_le_client(args, config, authenticator, installer) + le_client = _init_le_client(config, authenticator, installer) lineage, action = _auth_from_domains(le_client, config, domains) @@ -571,82 +671,331 @@ def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-lo else: display_ops.success_renewal(domains, action) - _suggest_donate() + _suggest_donation_if_appropriate(config) -def obtain_cert(args, config, plugins): - """Authenticate & obtain cert, but do not install it.""" - if args.domains and args.csr is not None: - # TODO: --csr could have a priority, when --domains is - # supplied, check if CSR matches given domains? - return "--domains and --csr are mutually exclusive" +def obtain_cert(config, plugins, lineage=None): + """Implements "certonly": authenticate & obtain cert, but do not install it.""" try: # installers are used in auth mode to determine domain names - installer, authenticator = choose_configurator_plugins(args, config, plugins, "certonly") - except errors.PluginSelectionError, e: - return e.message + installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") + except errors.PluginSelectionError as e: + logger.info("Could not choose appropriate plugin: %s", e) + raise # TODO: Handle errors from _init_le_client? - le_client = _init_le_client(args, config, authenticator, installer) + le_client = _init_le_client(config, authenticator, installer) # This is a special case; cert and chain are simply saved - if args.csr is not None: - certr, chain = le_client.obtain_certificate_from_csr(le_util.CSR( - file=args.csr[0], data=args.csr[1], form="der")) - cert_path, _, cert_fullchain = le_client.save_certificate( - certr, chain, args.cert_path, args.chain_path, args.fullchain_path) - _report_new_cert(cert_path, cert_fullchain) + if config.csr is not None: + assert lineage is None, "Did not expect a CSR with a RenewableCert" + csr, typ = config.actual_csr + certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ) + if config.dry_run: + logger.info( + "Dry run: skipping saving certificate to %s", config.cert_path) + else: + cert_path, _, cert_fullchain = le_client.save_certificate( + certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + _report_new_cert(cert_path, cert_fullchain) else: - domains = _find_domains(args, installer) - _auth_from_domains(le_client, config, domains) + domains = _find_domains(config, installer) + _auth_from_domains(le_client, config, domains, lineage) - _suggest_donate() + if config.dry_run: + _report_successful_dry_run(config) + elif config.verb == "renew": + if installer is None: + # Tell the user that the server was not restarted. + print("new certificate deployed without reload, fullchain is", + lineage.fullchain) + else: + # In case of a renewal, reload server to pick up new certificate. + # In principle we could have a configuration option to inhibit this + # from happening. + installer.restart() + print("new certificate deployed with reload of", + config.installer, "server; fullchain is", lineage.fullchain) + _suggest_donation_if_appropriate(config) -def install(args, config, plugins): +def install(config, plugins): """Install a previously obtained cert in a server.""" # XXX: Update for renewer/RenewableCert + # FIXME: be consistent about whether errors are raised or returned from + # this function ... try: - installer, _ = choose_configurator_plugins(args, config, - plugins, "install") - except errors.PluginSelectionError, e: + installer, _ = choose_configurator_plugins(config, plugins, "install") + except errors.PluginSelectionError as e: return e.message - domains = _find_domains(args, installer) - le_client = _init_le_client( - args, config, authenticator=None, installer=installer) - assert args.cert_path is not None # required=True in the subparser + domains = _find_domains(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + assert config.cert_path is not None # required=True in the subparser le_client.deploy_certificate( - domains, args.key_path, args.cert_path, args.chain_path, - args.fullchain_path) + domains, config.key_path, config.cert_path, config.chain_path, + config.fullchain_path) le_client.enhance_config(domains, config) -def revoke(args, config, unused_plugins): # TODO: coop with renewal config +def _restore_required_config_elements(config, renewalparams): + """Sets non-plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: parameters from the renewal + configuration file that defines this lineage + + """ + # string-valued items to add if they're present + for config_item in STR_CONFIG_ITEMS: + if config_item in renewalparams: + value = renewalparams[config_item] + # Unfortunately, we've lost type information from ConfigObj, + # so we don't know if the original was NoneType or str! + if value == "None": + value = None + setattr(config.namespace, config_item, value) + # int-valued items to add if they're present + for config_item in INT_CONFIG_ITEMS: + if config_item in renewalparams: + try: + value = int(renewalparams[config_item]) + setattr(config.namespace, config_item, value) + except ValueError: + raise errors.Error( + "Expected a numeric value for {0}".format(config_item)) + + +def _restore_plugin_configs(config, renewalparams): + """Sets plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: Parameters from the renewal + configuration file that defines this lineage + + """ + # Now use parser to get plugin-prefixed items with correct types + # XXX: the current approach of extracting only prefixed items + # related to the actually-used installer and authenticator + # works as long as plugins don't need to read plugin-specific + # variables set by someone else (e.g., assuming Apache + # configurator doesn't need to read webroot_ variables). + # Note: if a parameter that used to be defined in the parser is no + # longer defined, stored copies of that parameter will be + # deserialized as strings by this logic even if they were + # originally meant to be some other type. + plugin_prefixes = [renewalparams["authenticator"]] + if renewalparams.get("installer", None) is not None: + plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(plugin_prefixes): + for config_item, config_value in renewalparams.iteritems(): + if config_item.startswith(plugin_prefix + "_"): + # Avoid confusion when, for example, "csr = None" (avoid + # trying to read the file called "None") + # Should we omit the item entirely rather than setting + # its value to None? + if config_value == "None": + setattr(config.namespace, config_item, None) + continue + for action in _parser.parser._actions: # pylint: disable=protected-access + if action.type is not None and action.dest == config_item: + setattr(config.namespace, config_item, + action.type(config_value)) + break + else: + setattr(config.namespace, config_item, str(config_value)) + + +def _reconstitute(config, full_path): + """Try to instantiate a RenewableCert, updating config with relevant items. + + This is specifically for use in renewal and enforces several checks + and policies to ensure that we can try to proceed with the renwal + request. The config argument is modified by including relevant options + read from the renewal configuration file. + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param str full_path: Absolute path to the configuration file that + defines this lineage + + :returns: the RenewableCert object or None if a fatal error occurred + :rtype: `storage.RenewableCert` or NoneType + + """ + try: + renewal_candidate = storage.RenewableCert( + full_path, configuration.RenewerConfiguration(config)) + except (errors.CertStorageError, IOError): + logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + return None + if "renewalparams" not in renewal_candidate.configuration: + logger.warning("Renewal configuration file %s lacks " + "renewalparams. Skipping.", full_path) + return None + renewalparams = renewal_candidate.configuration["renewalparams"] + if "authenticator" not in renewalparams: + logger.warning("Renewal configuration file %s does not specify " + "an authenticator. Skipping.", full_path) + return None + # Now restore specific values along with their data types, if + # those elements are present. + try: + _restore_required_config_elements(config, renewalparams) + _restore_plugin_configs(config, renewalparams) + except (ValueError, errors.Error) as error: + logger.warning( + "An error occured while parsing %s. The error was %s. " + "Skipping the file.", full_path, error.message) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + return None + + # webroot_map is, uniquely, a dict, and the general-purpose + # configuration restoring logic is not able to correctly parse it + # from the serialized form. + if "webroot_map" in renewalparams: + setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) + + try: + domains = [le_util.enforce_domain_sanity(x) for x in + renewal_candidate.names()] + except errors.ConfigurationError as error: + logger.warning("Renewal configuration file %s references a cert " + "that contains an invalid domain name. The problem " + "was: %s. Skipping.", full_path, error) + return None + + setattr(config.namespace, "domains", domains) + return renewal_candidate + + +def _renewal_conf_files(config): + """Return /path/to/*.conf in the renewal conf directory""" + return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + + +def _renew_describe_results(config, renew_successes, renew_failures, + renew_skipped, parse_failures): + status = lambda x, msg: " " + "\n ".join(i + " (" + msg +")" for i in x) + if config.dry_run: + print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry") + print("** (The test certificates below have not been saved.)") + print() + if renew_skipped: + print("The following certs are not due for renewal yet:") + print(status(renew_skipped, "skipped")) + if not renew_successes and not renew_failures: + print("No renewals were attempted.") + elif renew_successes and not renew_failures: + print("Congratulations, all renewals succeeded. The following certs " + "have been renewed:") + print(status(renew_successes, "success")) + elif renew_failures and not renew_successes: + print("All renewal attempts failed. The following certs could not be " + "renewed:") + print(status(renew_failures, "failure")) + elif renew_failures and renew_successes: + print("The following certs were successfully renewed:") + print(status(renew_successes, "success")) + print("\nThe following certs could not be renewed:") + print(status(renew_failures, "failure")) + + if parse_failures: + print("\nAdditionally, the following renewal configuration files " + "were invalid: ") + print(status(parse_failures, "parsefail")) + + if config.dry_run: + print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry") + print("** (The test certificates above have not been saved.)") + + +def renew(config, unused_plugins): + """Renew previously-obtained certificates.""" + if config.domains != []: + raise errors.Error("Currently, the renew verb is only capable of " + "renewing all installed certificates that are due " + "to be renewed; individual domains cannot be " + "specified with this action. If you would like to " + "renew specific certificates, use the certonly " + "command. The renew verb may provide other options " + "for selecting certificates to renew in the future.") + if config.csr is not None: + raise errors.Error("Currently, the renew verb cannot be used when " + "specifying a CSR file. Please try the certonly " + "command instead.") + renewer_config = configuration.RenewerConfiguration(config) + renew_successes = [] + renew_failures = [] + renew_skipped = [] + parse_failures = [] + for renewal_file in _renewal_conf_files(renewer_config): + print("Processing " + renewal_file) + lineage_config = copy.deepcopy(config) + + # Note that this modifies config (to add back the configuration + # elements from within the renewal configuration file). + try: + renewal_candidate = _reconstitute(lineage_config, renewal_file) + except Exception as e: # pylint: disable=broad-except + logger.warning("Renewal configuration file %s produced an " + "unexpected error: %s. Skipping.", renewal_file, e) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + parse_failures.append(renewal_file) + continue + + try: + if renewal_candidate is None: + parse_failures.append(renewal_file) + else: + # XXX: ensure that each call here replaces the previous one + zope.component.provideUtility(lineage_config) + if _should_renew(lineage_config, renewal_candidate): + plugins = plugins_disco.PluginsRegistry.find_all() + obtain_cert(lineage_config, plugins, renewal_candidate) + renew_successes.append(renewal_candidate.fullchain) + else: + renew_skipped.append(renewal_candidate.fullchain) + except Exception as e: # pylint: disable=broad-except + # obtain_cert (presumably) encountered an unanticipated problem. + logger.warning("Attempting to renew cert from %s produced an " + "unexpected error: %s. Skipping.", renewal_file, e) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + renew_failures.append(renewal_candidate.fullchain) + + # Describe all the results + _renew_describe_results(config, renew_successes, renew_failures, + renew_skipped, parse_failures) + + +def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" # For user-agent construction config.namespace.installer = config.namespace.authenticator = "none" - if args.key_path is not None: # revocation by cert key + if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", - args.cert_path[0], args.key_path[0]) - key = jose.JWK.load(args.key_path[1]) + config.cert_path[0], config.key_path[0]) + key = jose.JWK.load(config.key_path[1]) else: # revocation by account key - logger.debug("Revoking %s using Account Key", args.cert_path[0]) - acc, _ = _determine_account(args, config) + logger.debug("Revoking %s using Account Key", config.cert_path[0]) + acc, _ = _determine_account(config) key = acc.key acme = client.acme_from_config_key(config, key) - cert = crypto_util.pyopenssl_load_certificate(args.cert_path[1])[0] + cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] acme.revoke(jose.ComparableX509(cert)) -def rollback(args, config, plugins): +def rollback(config, plugins): """Rollback server configuration changes made during install.""" - client.rollback(args.installer, args.checkpoints, config, plugins) + client.rollback(config.installer, config.checkpoints, config, plugins) -def config_changes(unused_args, config, unused_plugins): +def config_changes(config, unused_plugins): """Show changes made to server config during installation View checkpoints and associated configuration changes. @@ -655,15 +1004,15 @@ def config_changes(unused_args, config, unused_plugins): client.view_config_changes(config) -def plugins_cmd(args, config, plugins): # TODO: Use IDisplay rather than print +def plugins_cmd(config, plugins): # TODO: Use IDisplay rather than print """List server software plugins.""" - logger.debug("Expected interfaces: %s", args.ifaces) + logger.debug("Expected interfaces: %s", config.ifaces) - ifaces = [] if args.ifaces is None else args.ifaces + ifaces = [] if config.ifaces is None else config.ifaces filtered = plugins.visible().ifaces(ifaces) logger.debug("Filtered plugins: %r", filtered) - if not args.init and not args.prepare: + if not config.init and not config.prepare: print(str(filtered)) return @@ -671,7 +1020,7 @@ def plugins_cmd(args, config, plugins): # TODO: Use IDisplay rather than print verified = filtered.verify(ifaces) logger.debug("Verified plugins: %r", verified) - if not args.prepare: + if not config.prepare: print(str(verified)) return @@ -742,7 +1091,7 @@ class HelpfulArgumentParser(object): # Maps verbs/subcommands to the functions that implement them VERBS = {"auth": obtain_cert, "certonly": obtain_cert, "config_changes": config_changes, "everything": run, - "install": install, "plugins": plugins_cmd, + "install": install, "plugins": plugins_cmd, "renew": renew, "revoke": revoke, "rollback": rollback, "run": run} # List of topics for which additional help can be provided @@ -789,14 +1138,74 @@ class HelpfulArgumentParser(object): # Do any post-parsing homework here - # argparse seemingly isn't flexible enough to give us this behaviour easily... - if parsed_args.staging: - if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): - raise errors.Error("--server value conflicts with --staging") + # we get domains from -d, but also from the webroot map... + if parsed_args.webroot_map: + for domain in parsed_args.webroot_map.keys(): + if domain not in parsed_args.domains: + parsed_args.domains.append(domain) + + if parsed_args.staging or parsed_args.dry_run: + if (parsed_args.server not in + (flag_default("server"), constants.STAGING_URI)): + conflicts = ["--staging"] if parsed_args.staging else [] + conflicts += ["--dry-run"] if parsed_args.dry_run else [] + raise errors.Error("--server value conflicts with {0}".format( + " and ".join(conflicts))) + parsed_args.server = constants.STAGING_URI + if parsed_args.dry_run: + if self.verb not in ["certonly", "renew"]: + raise errors.Error("--dry-run currently only works with the " + "'certonly' or 'renew' subcommands") + parsed_args.break_my_certs = parsed_args.staging = True + + if parsed_args.csr: + self.handle_csr(parsed_args) + return parsed_args + def handle_csr(self, parsed_args): + """ + Process a --csr flag. This needs to happen early enough that the + webroot plugin can know about the calls to _process_domain + """ + try: + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") + typ = OpenSSL.crypto.FILETYPE_ASN1 + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + except OpenSSL.crypto.Error: + try: + e1 = traceback.format_exc() + typ = OpenSSL.crypto.FILETYPE_PEM + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="pem") + domains = crypto_util.get_sans_from_csr(csr.data, typ) + except OpenSSL.crypto.Error: + logger.debug("DER CSR parse error %s", e1) + logger.debug("PEM CSR parse error %s", traceback.format_exc()) + raise errors.Error("Failed to parse CSR file: {0}".format(parsed_args.csr[0])) + for d in domains: + _process_domain(parsed_args, d) + + for d in domains: + sanitised = le_util.enforce_domain_sanity(d) + if d.lower() != sanitised: + raise errors.ConfigurationError( + "CSR domain {0} needs to be sanitised to {1}.".format(d, sanitised)) + + if not domains: + # TODO: add CN to domains instead: + raise errors.Error( + "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain" + % parsed_args.csr[0]) + + parsed_args.actual_csr = (csr, typ) + csr_domains, config_domains = set(domains), set(parsed_args.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" + .format(", ".join(csr_domains), ", ".join(config_domains))) + def determine_verb(self): """Determines the verb/subcommand provided by the user. @@ -948,6 +1357,12 @@ def prepare_and_parse_args(plugins, args): helpful.add( None, "-t", "--text", dest="text_mode", action="store_true", help="Use the text output instead of the curses UI.") + helpful.add( + None, "-n", "--non-interactive", "--noninteractive", + dest="noninteractive_mode", action="store_true", + help="Run without ever asking for user input. This may require " + "additional command line flags; the client will try to explain " + "which ones are required if it finds one missing") helpful.add( None, "--register-unsafely-without-email", action="store_true", help="Specifying this flag enables registering an account with no " @@ -1078,6 +1493,8 @@ def prepare_and_parse_args(plugins, args): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) + global _parser # pylint: disable=global-statement + _parser = helpful return helpful.parse_args() @@ -1164,6 +1581,10 @@ def _paths_parser(helpful): add("testing", "--test-cert", "--staging", action='store_true', dest='staging', help='Use the staging server to obtain test (invalid) certs; equivalent' ' to --server ' + constants.STAGING_URI) + add("testing", "--dry-run", action="store_true", dest="dry_run", + help="Perform a test run of the client, obtaining test (invalid) certs" + " but not saving them to disk. This can currently only be used" + " with the 'certonly' subcommand.") def _plugins_parsing(helpful, plugins): @@ -1171,9 +1592,8 @@ def _plugins_parsing(helpful, plugins): "plugins", description="Let's Encrypt client supports an " "extensible plugins architecture. See '%(prog)s plugins' for a " "list of all installed plugins and their names. You can force " - "a particular plugin by setting options provided below. Further " - "down this help message you will find plugin-specific options " - "(prefixed by --{plugin_name}).") + "a particular plugin by setting options provided below. Running " + "--help will list flags specific to that plugin.") helpful.add( "plugins", "-a", "--authenticator", help="Authenticator plugin name.") helpful.add( @@ -1208,56 +1628,83 @@ def _plugins_parsing(helpful, plugins): "handle different domains; each domain will have the webroot path that" " preceded it. For instance: `-w /var/www/example -d example.com -d " "www.example.com -w /var/www/thing -d thing.net -d m.thing.net`") - parse_dict = lambda s: dict(json.loads(s)) # --webroot-map still has some awkward properties, so it is undocumented - helpful.add("webroot", "--webroot-map", default={}, type=parse_dict, - help=argparse.SUPPRESS) + helpful.add("webroot", "--webroot-map", default={}, action=WebrootMapProcessor, + help="JSON dictionary mapping domains to webroot paths; this " + "implies -d for each entry. You may need to escape this " + "from your shell. E.g.: --webroot-map " + """'{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' """ + "This option is merged with, but takes precedence over, " + "-w / -d entries. At present, if you put webroot-map in " + "a config file, it needs to be on a single line, like: " + 'webroot-map = {"example.com":"/var/www"}.') -class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring +class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring def __init__(self, *args, **kwargs): self.domain_before_webroot = False argparse.Action.__init__(self, *args, **kwargs) - def __call__(self, parser, config, webroot, option_string=None): + def __call__(self, parser, args, webroot, option_string=None): """ Keep a record of --webroot-path / -w flags during processing, so that we know which apply to which -d flags """ - if config.webroot_path is None: # first -w flag encountered - config.webroot_path = [] + if args.webroot_path is None: # first -w flag encountered + args.webroot_path = [] # if any --domain flags preceded the first --webroot-path flag, # apply that webroot path to those; subsequent entries in - # config.webroot_map are filled in by cli.DomainFlagProcessor - if config.domains: + # args.webroot_map are filled in by cli.DomainFlagProcessor + if args.domains: self.domain_before_webroot = True - for d in config.domains: - config.webroot_map.setdefault(d, webroot) + for d in args.domains: + args.webroot_map.setdefault(d, webroot) elif self.domain_before_webroot: - # FIXME if you set domains in a config file, you should get a different error + # FIXME if you set domains in a args file, you should get a different error # here, pointing you to --webroot-map raise errors.Error("If you specify multiple webroot paths, one of " "them must precede all domain flags") - config.webroot_path.append(webroot) + args.webroot_path.append(webroot) -class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring - def __call__(self, parser, config, domain_arg, option_string=None): - """ - Process a new -d flag, helping the webroot plugin construct a map of - {domain : webrootpath} if -w / --webroot-path is in use - """ - for domain in (d.strip() for d in domain_arg.split(",")): - if domain not in config.domains: - config.domains.append(domain) - # Each domain has a webroot_path of the most recent -w flag - if config.webroot_path: - config.webroot_map[domain] = config.webroot_path[-1] +def _process_domain(args_or_config, domain_arg, webroot_path=None): + """ + Process a new -d flag, helping the webroot plugin construct a map of + {domain : webrootpath} if -w / --webroot-path is in use + + :param args_or_config: may be an argparse args object, or a NamespaceConfig object + :param str domain_arg: a string representing 1+ domains, eg: "eg.is, example.com" + :param str webroot_path: (optional) the webroot_path for these domains + + """ + webroot_path = webroot_path if webroot_path else args_or_config.webroot_path + + for domain in (d.strip() for d in domain_arg.split(",")): + domain = le_util.enforce_domain_sanity(domain) + if domain not in args_or_config.domains: + args_or_config.domains.append(domain) + # Each domain has a webroot_path of the most recent -w flag + # unless it was explicitly included in webroot_map + if webroot_path: + args_or_config.webroot_map.setdefault(domain, webroot_path[-1]) -def setup_log_file_handler(args, logfile, fmt): +class WebrootMapProcessor(argparse.Action): # pylint: disable=missing-docstring + def __call__(self, parser, args, webroot_map_arg, option_string=None): + webroot_map = json.loads(webroot_map_arg) + for domains, webroot_path in webroot_map.iteritems(): + _process_domain(args, domains, [webroot_path]) + + +class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring + def __call__(self, parser, args, domain_arg, option_string=None): + """Just wrap _process_domain in argparseese.""" + _process_domain(args, domain_arg) + + +def setup_log_file_handler(config, logfile, fmt): """Setup file debug logging.""" - log_file_path = os.path.join(args.logs_dir, logfile) + log_file_path = os.path.join(config.logs_dir, logfile) handler = logging.handlers.RotatingFileHandler( log_file_path, maxBytes=2 ** 20, backupCount=10) # rotate on each invocation, rollover only possible when maxBytes @@ -1271,8 +1718,8 @@ def setup_log_file_handler(args, logfile, fmt): return handler, log_file_path -def _cli_log_handler(args, level, fmt): - if args.text_mode: +def _cli_log_handler(config, level, fmt): + if config.text_mode or config.noninteractive_mode or config.verb == "renew": handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) else: @@ -1283,13 +1730,13 @@ def _cli_log_handler(args, level, fmt): return handler -def setup_logging(args, cli_handler_factory, logfile): +def setup_logging(config, cli_handler_factory, logfile): """Setup logging.""" fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" - level = -args.verbose_count * 10 + level = -config.verbose_count * 10 file_handler, log_file_path = setup_log_file_handler( - args, logfile=logfile, fmt=fmt) - cli_handler = cli_handler_factory(args, level, fmt) + config, logfile=logfile, fmt=fmt) + cli_handler = cli_handler_factory(config, level, fmt) # TODO: use fileConfig? @@ -1302,12 +1749,12 @@ def setup_logging(args, cli_handler_factory, logfile): logger.info("Saving debug log to %s", log_file_path) -def _handle_exception(exc_type, exc_value, trace, args): +def _handle_exception(exc_type, exc_value, trace, config): """Logs exceptions and reports them to the user. - Args is used to determine how to display exceptions to the user. In - general, if args.debug is True, then the full exception and traceback is - shown to the user, otherwise it is suppressed. If args itself is None, + Config is used to determine how to display exceptions to the user. In + general, if config.debug is True, then the full exception and traceback is + shown to the user, otherwise it is suppressed. If config itself is None, then the traceback and exception is attempted to be written to a logfile. If this is successful, the traceback is suppressed, otherwise it is shown to the user. sys.exit is always called with a nonzero status. @@ -1318,8 +1765,8 @@ def _handle_exception(exc_type, exc_value, trace, args): os.linesep, "".join(traceback.format_exception(exc_type, exc_value, trace))) - if issubclass(exc_type, Exception) and (args is None or not args.debug): - if args is None: + if issubclass(exc_type, Exception) and (config is None or not config.debug): + if config is None: logfile = "letsencrypt.log" try: with open(logfile, "w") as logfd: @@ -1340,15 +1787,15 @@ def _handle_exception(exc_type, exc_value, trace, args): # acme.messages.Error: urn:acme:error:malformed :: The request message was # malformed :: Error creating new registration :: Validation of contact # mailto:none@longrandomstring.biz failed: Server failure at resolver - if ("urn:acme" in err and ":: " in err - and args.verbose_count <= flag_default("verbose_count")): + if (("urn:acme" in err and ":: " in err and + config.verbose_count <= flag_default("verbose_count"))): # prune ACME error code, we have a human description _code, _sep, err = err.partition(":: ") msg = "An unexpected error occurred:\n" + err + "Please see the " - if args is None: + if config is None: msg += "logfile '{0}' for more details.".format(logfile) else: - msg += "logfiles in {0} for more details.".format(args.logs_dir) + msg += "logfiles in {0} for more details.".format(config.logs_dir) sys.exit(msg) else: sys.exit("".join( @@ -1357,7 +1804,7 @@ def _handle_exception(exc_type, exc_value, trace, args): def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" - sys.excepthook = functools.partial(_handle_exception, args=None) + sys.excepthook = functools.partial(_handle_exception, config=None) # note: arg parser internally handles --help (and exits afterwards) plugins = plugins_disco.PluginsRegistry.find_all() @@ -1374,19 +1821,24 @@ def main(cli_args=sys.argv[1:]): # TODO: logs might contain sensitive data such as contents of the # private key! #525 le_util.make_or_verify_dir( - args.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) - setup_logging(args, _cli_log_handler, logfile='letsencrypt.log') + config.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) + setup_logging(config, _cli_log_handler, logfile='letsencrypt.log') logger.debug("letsencrypt version: %s", letsencrypt.__version__) - # do not log `args`, as it contains sensitive data (e.g. revoke --key)! + # do not log `config`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) logger.debug("Discovered plugins: %r", plugins) - sys.excepthook = functools.partial(_handle_exception, args=args) + sys.excepthook = functools.partial(_handle_exception, config=config) # Displayer - if args.text_mode: + if config.noninteractive_mode: + displayer = display_util.NoninteractiveDisplay(sys.stdout) + elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) + elif config.verb == "renew": + config.noninteractive_mode = True + displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() zope.component.provideUtility(displayer) @@ -1407,7 +1859,7 @@ def main(cli_args=sys.argv[1:]): # "{0}Root is required to run letsencrypt. Please use sudo.{0}" # .format(os.linesep)) - return args.func(args, config, plugins) + return config.func(config, plugins) if __name__ == "__main__": err_string = main() diff --git a/letsencrypt/client.py b/letsencrypt/client.py index c2dfca1bf..9dfa70e8d 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -146,7 +146,7 @@ def perform_registration(acme, config): """ try: return acme.register(messages.NewRegistration.from_data(email=config.email)) - except messages.Error, e: + except messages.Error as e: err = repr(e) if "MX record" in err or "Validation of contact mailto" in err: config.namespace.email = display_ops.get_email(more=True, invalid=True) @@ -195,7 +195,8 @@ class Client(object): else: self.auth_handler = None - def _obtain_certificate(self, domains, csr): + def obtain_certificate_from_csr(self, domains, csr, + typ=OpenSSL.crypto.FILETYPE_ASN1): """Obtain certificate. Internal function with precondition that `domains` are @@ -223,33 +224,18 @@ class Client(object): authzr = self.auth_handler.get_authorizations(domains) certr = self.acme.request_issuance( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr.data)), + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(typ, csr.data)), authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, csr): - """Obtain certficiate from CSR. - - :param .le_util.CSR csr: DER-encoded Certificate Signing - Request. - - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). - :rtype: tuple - - """ - return self._obtain_certificate( - # TODO: add CN to domains? - crypto_util.get_sans_from_csr( - csr.data, OpenSSL.crypto.FILETYPE_ASN1), csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. `.register` must be called before `.obtain_certificate` - :param set domains: domains to get a certificate + :param list domains: domains to get a certificate :returns: `.CertificateResource`, certificate chain (as returned by `.fetch_chain`), and newly generated private key @@ -263,7 +249,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - return self._obtain_certificate(domains, csr) + (key, csr) + return self.obtain_certificate_from_csr(domains, csr) + (key, csr) def obtain_and_enroll_certificate(self, domains): """Obtain and enroll certificate. @@ -276,34 +262,28 @@ class Client(object): :param plugins: A PluginsFactory object. :returns: A new :class:`letsencrypt.storage.RenewableCert` instance - referred to the enrolled cert lineage, or False if the cert could - not be obtained. + referred to the enrolled cert lineage, False if the cert could not + be obtained, or None if doing a successful dry run. """ certr, chain, key, _ = self.obtain_certificate(domains) - # XXX: We clearly need a more general and correct way of getting - # options into the configobj for the RenewableCert instance. - # This is a quick-and-dirty way to do it to allow integration - # testing to start. (Note that the config parameter to new_lineage - # ideally should be a ConfigObj, but in this case a dict will be - # accepted in practice.) - params = vars(self.config.namespace) - config = {} - cli_config = configuration.RenewerConfiguration(self.config.namespace) - - if (cli_config.config_dir != constants.CLI_DEFAULTS["config_dir"] or - cli_config.work_dir != constants.CLI_DEFAULTS["work_dir"]): + if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or + self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): logger.warning( "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - lineage = storage.RenewableCert.new_lineage( - domains[0], OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), - params, config, cli_config) - return lineage + if self.config.dry_run: + logger.info("Dry run: Skipping creating new lineage for %s", + domains[0]) + return None + else: + return storage.RenewableCert.new_lineage( + domains[0], OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), + key.pem, crypto_util.dump_pyopenssl_chain(chain), + configuration.RenewerConfiguration(self.config.namespace)) def save_certificate(self, certr, chain_cert, cert_path, chain_path, fullchain_path): diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index afd5edbe4..2bbf1b019 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -1,4 +1,5 @@ """Let's Encrypt user-supplied configuration.""" +import copy import os import urlparse @@ -78,6 +79,12 @@ class NamespaceConfig(object): return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) + def __deepcopy__(self, _memo): + # Work around https://bugs.python.org/issue1515 for py26 tests :( :( + # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 + new_ns = copy.deepcopy(self.namespace) + return type(self)(new_ns) + class RenewerConfiguration(object): """Configuration wrapper for renewer.""" @@ -124,4 +131,5 @@ def check_config_sanity(config): # Domain checks if config.namespace.domains is not None: for domain in config.namespace.domains: - le_util.check_domain_sanity(domain) + # This may be redundant, but let's be paranoid + le_util.enforce_domain_sanity(domain) diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index a1dccd1ea..402f5e9a1 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -37,7 +37,9 @@ STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" RENEWER_DEFAULTS = dict( renewer_enabled="yes", renew_before_expiry="30 days", - deploy_before_expiry="20 days", + # This value should ensure that there is never a deployment delay by + # default. + deploy_before_expiry="99 years", ) """Defaults for renewer script.""" diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index 730c32398..76265a739 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -53,8 +53,8 @@ def init_save_key(key_size, key_dir, keyname="key-letsencrypt.pem"): config.strict_permissions) key_f, key_path = le_util.unique_file( os.path.join(key_dir, keyname), 0o600) - key_f.write(key_pem) - key_f.close() + with key_f: + key_f.write(key_pem) logger.info("Generating key (%d bits): %s", key_size, key_path) diff --git a/letsencrypt/display/enhancements.py b/letsencrypt/display/enhancements.py index c56198161..39def1651 100644 --- a/letsencrypt/display/enhancements.py +++ b/letsencrypt/display/enhancements.py @@ -48,7 +48,7 @@ def redirect_by_default(): code, selection = util(interfaces.IDisplay).menu( "Please choose whether HTTPS access is required or optional.", - choices) + choices, default=0, cli_flag="--redirect / --no-redirect") if code != display_util.OK: return False diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index a82f84d6d..f0dec8b06 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -31,8 +31,8 @@ def choose_plugin(prepared, question): for plugin_ep in prepared] while True: - code, index = util(interfaces.IDisplay).menu( - question, opts, help_label="More Info") + disp = util(interfaces.IDisplay) + code, index = disp.menu(question, opts, help_label="More Info") if code == display_util.OK: plugin_ep = prepared[index] @@ -74,6 +74,17 @@ def pick_plugin(config, default, plugins, question, ifaces): # throw more UX-friendly error if default not in plugins filtered = plugins.filter(lambda p_ep: p_ep.name == default) else: + if config.noninteractive_mode: + # it's really bad to auto-select the single available plugin in + # non-interactive mode, because an update could later add a second + # available plugin + raise errors.MissingCommandlineFlag( + "Missing command line flags. For non-interactive execution, " + "you will need to specify a plugin on the command line. Run " + "with '--help plugins' to see a list of options, and see " + "https://eff.org/letsencrypt-plugins for more detail on what " + "the plugins do and how to use them.") + filtered = plugins.visible().ifaces(ifaces) filtered.init(config) @@ -143,7 +154,12 @@ def get_email(more=False, invalid=False): msg += ('\n\nIf you really want to skip this, you can run the client with ' '--register-unsafely-without-email but make sure you backup your ' 'account key from /etc/letsencrypt/accounts\n\n') - code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) + try: + code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) + except errors.MissingCommandlineFlag: + msg = ("You should register before running non-interactively, or provide --agree-tos" + " and --email flags") + raise errors.MissingCommandlineFlag(msg) if code == display_util.OK: if le_util.safe_email(email): @@ -197,7 +213,8 @@ def choose_names(installer): "specify ServerNames in your config files in order to allow for " "accurate installation of your certificate.{0}" "If you do use the default vhost, you may specify the name " - "manually. Would you like to continue?{0}".format(os.linesep)) + "manually. Would you like to continue?{0}".format(os.linesep), + default=True) if manual: return _choose_names_manually() @@ -222,8 +239,7 @@ def get_valid_domains(domains): valid_domains = [] for domain in domains: try: - le_util.check_domain_sanity(domain) - valid_domains.append(domain) + valid_domains.append(le_util.enforce_domain_sanity(domain)) except errors.ConfigurationError: continue return valid_domains @@ -242,7 +258,7 @@ def _filter_names(names): """ code, names = util(interfaces.IDisplay).checklist( "Which names would you like to activate HTTPS for?", - tags=names) + tags=names, cli_flag="--domains") return code, [str(s) for s in names] @@ -250,7 +266,8 @@ def _choose_names_manually(): """Manually input names for those without an installer.""" code, input_ = util(interfaces.IDisplay).input( - "Please enter in your domain name(s) (comma and/or space separated) ") + "Please enter in your domain name(s) (comma and/or space separated) ", + cli_flag="--domains") if code == display_util.OK: invalid_domains = dict() @@ -264,9 +281,9 @@ def _choose_names_manually(): "supported.{0}{0}Would you like to re-enter the " "names?{0}").format(os.linesep) - for domain in domain_list: + for i, domain in enumerate(domain_list): try: - le_util.check_domain_sanity(domain) + domain_list[i] = le_util.enforce_domain_sanity(domain) except errors.ConfigurationError as e: invalid_domains[domain] = e.message diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 01a8cbc92..976a2afdf 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -6,7 +6,7 @@ import dialog import zope.interface from letsencrypt import interfaces - +from letsencrypt import errors WIDTH = 72 HEIGHT = 20 @@ -21,6 +21,20 @@ CANCEL = "cancel" HELP = "help" """Display exit code when for when the user requests more help.""" +def _wrap_lines(msg): + """Format lines nicely to 80 chars. + + :param str msg: Original message + + :returns: Formatted message respecting newlines in message + :rtype: str + + """ + lines = msg.splitlines() + fixed_l = [] + for line in lines: + fixed_l.append(textwrap.fill(line, 80)) + return os.linesep.join(fixed_l) class NcursesDisplay(object): """Ncurses-based display.""" @@ -49,8 +63,8 @@ class NcursesDisplay(object): """ self.dialog.msgbox(message, height, width=self.width) - def menu(self, message, choices, - ok_label="OK", cancel_label="Cancel", help_label=""): + def menu(self, message, choices, ok_label="OK", cancel_label="Cancel", + help_label="", **unused_kwargs): """Display a menu. :param str message: title of menu @@ -61,10 +75,11 @@ class NcursesDisplay(object): :param str ok_label: label of the OK button :param str help_label: label of the help button + :param dict unused_kwargs: absorbs default / cli_args - :returns: tuple of the form (`code`, `tag`) where - `code` - `str` display_util exit code - `tag` - `int` index corresponding to the item chosen + :returns: tuple of the form (`code`, `index`) where + `code` - int display exit code + `int` - index of the selected item :rtype: tuple """ @@ -97,20 +112,21 @@ class NcursesDisplay(object): (str(i), choice) for i, choice in enumerate(choices, 1) ] # pylint: disable=star-args - code, tag = self.dialog.menu(message, **menu_options) + code, index = self.dialog.menu(message, **menu_options) if code == CANCEL: return code, -1 - return code, int(tag) - 1 + return code, int(index) - 1 - def input(self, message): + def input(self, message, **unused_kwargs): """Display an input box to the user. :param str message: Message to display that asks for input. + :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, string) where + :returns: tuple of the form (`code`, `string`) where `code` - int display exit code `string` - input entered by the user @@ -122,7 +138,7 @@ class NcursesDisplay(object): return self.dialog.inputbox(message, width=self.width, height=height) - def yesno(self, message, yes_label="Yes", no_label="No"): + def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs): """Display a Yes/No dialog box. Yes and No label must begin with different letters. @@ -130,6 +146,7 @@ class NcursesDisplay(object): :param str message: message to display to user :param str yes_label: label on the "yes" button :param str no_label: label on the "no" button + :param dict _kwargs: absorbs default / cli_args :returns: if yes_label was selected :rtype: bool @@ -139,16 +156,17 @@ class NcursesDisplay(object): message, self.height, self.width, yes_label=yes_label, no_label=no_label) - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default_status=True, **unused_kwargs): """Displays a checklist. :param message: Message to display before choices :param list tags: where each is of type :class:`str` len(tags) > 0 :param bool default_status: If True, items are in a selected state by default. + :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, list_tags) where + :returns: tuple of the form (`code`, `list_tags`) where `code` - int display exit code `list_tags` - list of str tags selected by the user @@ -178,15 +196,15 @@ class FileDisplay(object): """ side_frame = "-" * 79 - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( line=os.linesep, frame=side_frame, msg=message)) if pause: raw_input("Press Enter to Continue") - def menu(self, message, choices, - ok_label="", cancel_label="", help_label=""): + def menu(self, message, choices, ok_label="", cancel_label="", + help_label="", **unused_kwargs): # pylint: disable=unused-argument """Display a menu. @@ -197,10 +215,12 @@ class FileDisplay(object): :param choices: Menu lines, len must be > 0 :type choices: list of tuples (tag, item) or list of descriptions (tags will be enumerated) + :param dict _kwargs: absorbs default / cli_args + + :returns: tuple of (`code`, `index`) where + `code` - str display exit code + `index` - int index of the user's selection - :returns: tuple of the form (code, tag) where - code - int display exit code - tag - str corresponding to the item chosen :rtype: tuple """ @@ -210,11 +230,12 @@ class FileDisplay(object): return code, selection - 1 - def input(self, message): + def input(self, message, **unused_kwargs): # pylint: disable=no-self-use """Accept input from the user. :param str message: message to display to the user + :param dict _kwargs: absorbs default / cli_args :returns: tuple of (`code`, `input`) where `code` - str display exit code @@ -230,7 +251,7 @@ class FileDisplay(object): else: return OK, ans - def yesno(self, message, yes_label="Yes", no_label="No"): + def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs): """Query the user with a yes/no question. Yes and No label must begin with different letters, and must contain at @@ -239,6 +260,7 @@ class FileDisplay(object): :param str message: question for the user :param str yes_label: Label of the "Yes" parameter :param str no_label: Label of the "No" parameter + :param dict _kwargs: absorbs default / cli_args :returns: True for "Yes", False for "No" :rtype: bool @@ -246,7 +268,7 @@ class FileDisplay(object): """ side_frame = ("-" * 79) + os.linesep - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write("{0}{frame}{msg}{0}{frame}".format( os.linesep, frame=side_frame, msg=message)) @@ -265,13 +287,14 @@ class FileDisplay(object): ans.startswith(no_label[0].upper())): return False - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default_status=True, **unused_kwargs): # pylint: disable=unused-argument """Display a checklist. :param str message: Message to display to user :param list tags: `str` tags to select, len(tags) > 0 :param bool default_status: Not used for FileDisplay + :param dict _kwargs: absorbs default / cli_args :returns: tuple of (`code`, `tags`) where `code` - str display exit code @@ -352,21 +375,6 @@ class FileDisplay(object): self.outfile.write(side_frame) - def _wrap_lines(self, msg): # pylint: disable=no-self-use - """Format lines nicely to 80 chars. - - :param str msg: Original message - - :returns: Formatted message respecting newlines in message - :rtype: str - - """ - lines = msg.splitlines() - fixed_l = [] - for line in lines: - fixed_l.append(textwrap.fill(line, 80)) - - return os.linesep.join(fixed_l) def _get_valid_int_ans(self, max_): """Get a numerical selection. @@ -403,6 +411,118 @@ class FileDisplay(object): return OK, selection +class NoninteractiveDisplay(object): + """An iDisplay implementation that never asks for interactive user input""" + + zope.interface.implements(interfaces.IDisplay) + + def __init__(self, outfile): + super(NoninteractiveDisplay, self).__init__() + self.outfile = outfile + + def _interaction_fail(self, message, cli_flag, extra=""): + "Error out in case of an attempt to interact in noninteractive mode" + msg = "Missing command line flag or config entry for this setting:\n" + msg += message + if extra: + msg += "\n" + extra + if cli_flag: + msg += "\n\n(You can set this with the {0} flag)".format(cli_flag) + raise errors.MissingCommandlineFlag(msg) + + def notification(self, message, height=10, pause=False): + # pylint: disable=unused-argument + """Displays a notification without waiting for user acceptance. + + :param str message: Message to display to stdout + :param int height: No effect for NoninteractiveDisplay + :param bool pause: The NoninteractiveDisplay waits for no keyboard + + """ + side_frame = "-" * 79 + message = _wrap_lines(message) + self.outfile.write( + "{line}{frame}{line}{msg}{line}{frame}{line}".format( + line=os.linesep, frame=side_frame, msg=message)) + + def menu(self, message, choices, ok_label=None, cancel_label=None, + help_label=None, default=None, cli_flag=None): + # pylint: disable=unused-argument,too-many-arguments + """Avoid displaying a menu. + + :param str message: title of menu + :param choices: Menu lines, len must be > 0 + :type choices: list of tuples (tag, item) or + list of descriptions (tags will be enumerated) + :param int default: the default choice + :param dict kwargs: absorbs various irrelevant labelling arguments + + :returns: tuple of (`code`, `index`) where + `code` - str display exit code + `index` - int index of the user's selection + :rtype: tuple + :raises errors.MissingCommandlineFlag: if there was no default + + """ + if default is None: + self._interaction_fail(message, cli_flag, "Choices: " + repr(choices)) + + return OK, default + + def input(self, message, default=None, cli_flag=None): + """Accept input from the user. + + :param str message: message to display to the user + + :returns: tuple of (`code`, `input`) where + `code` - str display exit code + `input` - str of the user's input + :rtype: tuple + :raises errors.MissingCommandlineFlag: if there was no default + + """ + if default is None: + self._interaction_fail(message, cli_flag) + else: + return OK, default + + + def yesno(self, message, yes_label=None, no_label=None, default=None, cli_flag=None): + # pylint: disable=unused-argument + """Decide Yes or No, without asking anybody + + :param str message: question for the user + :param dict kwargs: absorbs yes_label, no_label + + :raises errors.MissingCommandlineFlag: if there was no default + :returns: True for "Yes", False for "No" + :rtype: bool + + """ + if default is None: + self._interaction_fail(message, cli_flag) + else: + return default + + def checklist(self, message, tags, default=None, cli_flag=None, **kwargs): + # pylint: disable=unused-argument + """Display a checklist. + + :param str message: Message to display to user + :param list tags: `str` tags to select, len(tags) > 0 + :param dict kwargs: absorbs default_status arg + + :returns: tuple of (`code`, `tags`) where + `code` - str display exit code + `tags` - list of selected tags + :rtype: tuple + + """ + if default is None: + self._interaction_fail(message, cli_flag, "? ".join(tags)) + else: + return OK, default + def separate_list_input(input_): """Separate a comma or space separated list. diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index 1358d1048..99bb29d9d 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -102,3 +102,8 @@ class StandaloneBindError(Error): class ConfigurationError(Error): """Configuration sanity error.""" + +# NoninteractiveDisplay iDisplay plugin error: + +class MissingCommandlineFlag(Error): + """A command line argument was missing in noninteractive usage""" diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index c8a725fde..db5d2c5e8 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -365,8 +365,8 @@ class IDisplay(zope.interface.Interface): """ - def menu(message, choices, - ok_label="OK", cancel_label="Cancel", help_label=""): + def menu(message, choices, ok_label="OK", # pylint: disable=too-many-arguments + cancel_label="Cancel", help_label="", default=None, cli_flag=None): """Displays a generic menu. :param str message: message to display @@ -377,14 +377,19 @@ class IDisplay(zope.interface.Interface): :param str ok_label: label for OK button :param str cancel_label: label for Cancel button :param str help_label: label for Help button + :param int default: default (non-interactive) choice from the menu + :param str cli_flag: to automate choice from the menu, eg "--keep" :returns: tuple of (`code`, `index`) where `code` - str display exit code `index` - int index of the user's selection + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ - def input(message): + def input(message, default=None, cli_args=None): """Accept input from the user. :param str message: message to display to the user @@ -394,27 +399,45 @@ class IDisplay(zope.interface.Interface): `input` - str of the user's input :rtype: tuple + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ - def yesno(message, yes_label="Yes", no_label="No"): + def yesno(message, yes_label="Yes", no_label="No", default=None, + cli_args=None): """Query the user with a yes/no question. Yes and No label must begin with different letters. :param str message: question for the user + :param str default: default (non-interactive) choice from the menu + :param str cli_flag: to automate choice from the menu, eg "--redirect / --no-redirect" :returns: True for "Yes", False for "No" :rtype: bool + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ - def checklist(message, tags, default_state): + def checklist(message, tags, default_state, default=None, cli_args=None): """Allow for multiple selections from a menu. :param str message: message to display to the user :param list tags: where each is of type :class:`str` len(tags) > 0 - :param bool default_status: If True, items are in a selected state by - default. + :param bool default_status: If True, items are in a selected state by default. + :param str default: default (non-interactive) state of the checklist + :param str cli_flag: to automate choice from the menu, eg "--domains" + + :returns: tuple of the form (code, list_tags) where + `code` - int display exit code + `list_tags` - list of str tags selected by the user + :rtype: tuple + + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set """ diff --git a/letsencrypt/le_util.py b/letsencrypt/le_util.py index 64295a80f..527c9bdae 100644 --- a/letsencrypt/le_util.py +++ b/letsencrypt/le_util.py @@ -285,31 +285,37 @@ def add_deprecated_argument(add_argument, argument_name, nargs): help=argparse.SUPPRESS, nargs=nargs) -def check_domain_sanity(domain): +def enforce_domain_sanity(domain): """Method which validates domain value and errors out if the requirements are not met. :param domain: Domain to check - :type domains: `string` + :type domains: `str` or `unicode` :raises ConfigurationError: for invalid domains and cases where Let's Encrypt currently will not issue certificates + :returns: The domain cast to `str`, with ASCII-only contents + :rtype: str """ # Check if there's a wildcard domain if domain.startswith("*."): raise errors.ConfigurationError( - "Wildcard domains are not supported") + "Wildcard domains are not supported: {0}".format(domain)) # Punycode if "xn--" in domain: raise errors.ConfigurationError( - "Punycode domains are not presently supported") + "Punycode domains are not presently supported: {0}".format(domain)) # Unicode try: - domain.encode('ascii') + domain = domain.encode('ascii').lower() except UnicodeDecodeError: raise errors.ConfigurationError( - "Internationalized domain names are not presently supported") + "Internationalized domain names are not presently supported: {0}" + .format(domain)) + + # Remove trailing dot + domain = domain[:-1] if domain.endswith('.') else domain # FQDN checks from # http://www.mkyong.com/regular-expressions/domain-name-regular-expression-example/ @@ -317,4 +323,5 @@ def check_domain_sanity(domain): # first and last char is not "-" fqdn = re.compile("^((?!-)[A-Za-z0-9-]{1,63}(? +staging = True +prepare = False +work_dir = /var/lib/letsencrypt +tos = False +init = False +http01_port = 80 +duplicate = False +noninteractive_mode = True +key_path = None +nginx = False +nginx_server_root = /etc/nginx +fullchain_path = /home/ubuntu/letsencrypt/chain.pem +email = None +csr = None +agree_dev_preview = None +redirect = None +verb = certonly +verbose_count = -3 +config_file = None +renew_by_default = False +hsts = False +apache_handle_sites = True +authenticator = standalone +domains = isnot.org, +rsa_key_size = 2048 +apache_challenge_location = /etc/apache2 +checkpoints = 1 +manual_test_mode = False +apache = False +cert_path = /home/ubuntu/letsencrypt/cert.pem +webroot_path = None +reinstall = False +expand = False +strict_permissions = False +apache_server_root = /etc/apache2 +account = None +dry_run = False +manual_public_ip_logging_ok = False +chain_path = /home/ubuntu/letsencrypt/chain.pem +break_my_certs = False +standalone = True +manual = False +server = https://acme-staging.api.letsencrypt.org/directory +standalone_supported_challenges = "tls-sni-01,http-01" +webroot = False +os_packages_only = False +apache_init_script = None +user_agent = None +apache_le_vhost_ext = -le-ssl.conf +debug = False +tls_sni_01_port = 443 +logs_dir = /var/log/letsencrypt +apache_vhost_root = /etc/apache2/sites-available +configurator = None +[[webroot_map]] diff --git a/letsencrypt/tests/testdata/webrootconftest.ini b/letsencrypt/tests/testdata/webrootconftest.ini new file mode 100644 index 000000000..de3bd98a6 --- /dev/null +++ b/letsencrypt/tests/testdata/webrootconftest.ini @@ -0,0 +1,3 @@ +webroot +webroot-path = /tmp +domains = eg.com, eg2.com diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index 3f62d7cad..000f86c31 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.1.dev0' +version = '0.4.0.dev0' install_requires = [ 'setuptools', # pkg_resources diff --git a/setup.py b/setup.py index 647bfb8d2..b094e2d04 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,10 @@ version = meta['version'] # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), - 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 + # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but + # saying so here causes a runtime error against our temporary fork of 0.9.3 + # in which we added 2.6 support (see #2243), so we relax the requirement. + 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', @@ -127,7 +130,6 @@ setup( entry_points={ 'console_scripts': [ 'letsencrypt = letsencrypt.cli:main', - 'letsencrypt-renewer = letsencrypt.renewer:main', ], 'letsencrypt.plugins': [ 'manual = letsencrypt.plugins.manual:Authenticator', diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 53996cd20..29618b97f 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -20,13 +20,19 @@ else readlink="readlink" fi -common() { - letsencrypt_test \ +common_no_force_renew() { + letsencrypt_test_no_force_renew \ --authenticator standalone \ --installer null \ "$@" } +common() { + common_no_force_renew \ + --renew-by-default \ + "$@" +} + common --domains le1.wtf --standalone-supported-challenges tls-sni-01 auth common --domains le2.wtf --standalone-supported-challenges http-01 run common -a manual -d le.wtf auth @@ -44,20 +50,27 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" -# the following assumes that Boulder issues certificates for less than -# 10 years, otherwise renewal will not take place -cat < "$root/conf/renewer.conf" -renew_before_expiry = 10 years -deploy_before_expiry = 10 years -EOF -letsencrypt-renewer $store_flags -dir="$root/conf/archive/le1.wtf" -for x in cert chain fullchain privkey; -do - latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" - live="$($readlink -f "$root/conf/live/le1.wtf/${x}.pem")" - [ "${dir}/${latest}" = "$live" ] # renewer fails this test -done +CheckCertCount() { + CERTCOUNT=`ls "${root}/conf/archive/le.wtf/cert"* | wc -l` + if [ "$CERTCOUNT" -ne "$1" ] ; then + echo Wrong cert count, not "$1" `ls "${root}/conf/archive/le.wtf/"*` + exit 1 + fi +} + +CheckCertCount 1 +# This won't renew (because it's not time yet) +common_no_force_renew renew +CheckCertCount 1 + +# --renew-by-default is used, so renewal should occur +common renew +CheckCertCount 2 + +# This will renew because the expiry is less than 10 years from now +sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" +common_no_force_renew renew +CheckCertCount 3 # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 4572b0fb3..e86d087cb 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -12,6 +12,12 @@ store_flags="$store_flags --logs-dir $root/logs" export root store_flags letsencrypt_test () { + letsencrypt_test_no_force_renew \ + --renew-by-default \ + "$@" +} + +letsencrypt_test_no_force_renew () { letsencrypt \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ @@ -19,11 +25,10 @@ letsencrypt_test () { --http-01-port 5002 \ --manual-test-mode \ $store_flags \ - --text \ + --non-interactive \ --no-redirect \ --agree-tos \ --register-unsafely-without-email \ - --renew-by-default \ --debug \ -vvvvvvv \ "$@" diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index dee6968c3..378670071 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -139,7 +139,15 @@ def make_instance(instance_name, time.sleep(1.0) # give instance a name - new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + try: + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + except botocore.exceptions.ClientError as e: + if "InvalidInstanceID.NotFound" in str(e): + # This seems to be ephemeral... retry + time.sleep(1) + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + else: + raise return new_instance def terminate_and_clean(instances): diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 4032e2195..940cc36c6 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -35,22 +35,16 @@ then #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/ fi -# run letsencrypt-apache2 via letsencrypt-auto +# Run letsencrypt-apache2. cd letsencrypt -export SUDO=sudo -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO bootstrap/_deb_common.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO bootstrap/_rpm_common.sh -else - echo "Dont have bootstrapping for this OS!" - exit 1 +echo "Bootstrapping dependencies..." +letsencrypt-auto-source/letsencrypt-auto --os-packages-only +if [ $? -ne 0 ] ; then + exit 1 fi -bootstrap/dev/venv.sh +tools/venv.sh sudo venv/bin/letsencrypt -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index b7849755a..0ad0d6081 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -8,12 +8,28 @@ cd letsencrypt SAVE="$PIP_EXTRA_INDEX_URL" unset PIP_EXTRA_INDEX_URL export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" -./letsencrypt-auto -v --debug --version + +#OLD_LEAUTO="https://raw.githubusercontent.com/letsencrypt/letsencrypt/5747ab7fd9641986833bad474d71b46a8c589247/letsencrypt-auto" + + +if ! command -v git ; then + if [ "$OS_TYPE" = "ubuntu" ] ; then + sudo apt-get update + fi + if ! ( sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then + echo git installation failed! + exit 1 + fi +fi +BRANCH=`git rev-parse --abbrev-ref HEAD` +git checkout v0.1.0 +./letsencrypt-auto -v --debug --version unset PIP_INDEX_URL export PIP_EXTRA_INDEX_URL="$SAVE" -if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then +git checkout -f "$BRANCH" +if ! ./letsencrypt-auto -v --debug --version | grep 0.3.0 ; then echo upgrade appeared to fail exit 1 fi diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh new file mode 100755 index 000000000..d90ae9ab6 --- /dev/null +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -0,0 +1,55 @@ +#!/bin/bash -x + +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution + +# run letsencrypt-apache2 via letsencrypt-auto +cd letsencrypt + +export SUDO=sudo +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO bootstrap/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO bootstrap/_rpm_common.sh +else + echo "Dont have bootstrapping for this OS!" + exit 1 +fi + +bootstrap/dev/venv.sh +sudo venv/bin/letsencrypt certonly --debug --standalone -t --agree-dev-preview --agree-tos \ + --renew-by-default --redirect --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -v +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +if [ "$OS_TYPE" = "ubuntu" ] ; then + venv/bin/tox -e apacheconftest +else + echo Not running hackish apache tests on $OS_TYPE +fi + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +sudo venv/bin/letsencrypt renew --renew-by-default + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + +ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +# return error if any of the subtests failed +if [ "$FAIL" = 1 ] ; then + exit 1 +fi diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index f7f325d5c..4c2eb429e 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -6,71 +6,15 @@ VENV_NAME="venv" LEA_PATH=./letsencrypt/ VENV_PATH=${LEA_PATH/$VENV_NAME} VENV_BIN=${VENV_PATH}/bin -BOOTSTRAP=${LEA_PATH}/bootstrap -SUDO=sudo - -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi - fi -} # virtualenv call is not idempotent: it overwrites pip upgraded in # later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh -elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." - exit 1 - fi -elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" -elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" -elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" -elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root -elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" -fi -echo "Bootstrapped!" +"$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt -./bootstrap/dev/venv.sh +./tools/venv.sh PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ $PYVER -eq 26 ] ; then diff --git a/bootstrap/dev/_venv_common.sh b/tools/_venv_common.sh similarity index 100% rename from bootstrap/dev/_venv_common.sh rename to tools/_venv_common.sh diff --git a/tools/release.sh b/tools/release.sh index d7dc4b6c6..83b57657f 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -81,21 +81,6 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# and that it's signed correctly -if ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; then - echo Failed letsencrypt-auto signature check on "$RELEASE_BRANCH" - echo please fix that and re-run - exit 1 -else - echo Signature check on letsencrypt-auto successful -fi - - SetVersion() { ver="$1" for pkg_dir in $SUBPKGS letsencrypt-compatibility-test @@ -110,9 +95,6 @@ SetVersion() { } SetVersion "$version" -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" \ - --sign --message "Release $version" "$tag" echo "Preparing sdists and wheels" for pkg_dir in . $SUBPKGS @@ -151,8 +133,12 @@ virtualenv --no-site-packages ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip -# Now, use our local PyPI +# Now, use our local PyPI. Disable cache so we get the correct KGS even if we +# (or our dependencies) have conditional dependencies implemented with if +# statements in setup.py and we have cached wheels lying around that would +# cause those ifs to not be evaluated. pip install \ + --no-cache-dir \ --extra-index-url http://localhost:$PORT \ letsencrypt $SUBPKGS # stop local PyPI @@ -175,6 +161,21 @@ for module in letsencrypt $subpkgs_modules ; do done deactivate +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; do + read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" +done + +git diff --cached +git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +git tag --local-user "$RELEASE_GPG_KEY" \ + --sign --message "Release $version" "$tag" + cd .. echo Now in $PWD name=${root_without_le%.*} diff --git a/bootstrap/dev/venv.sh b/tools/venv.sh similarity index 100% rename from bootstrap/dev/venv.sh rename to tools/venv.sh diff --git a/bootstrap/dev/venv3.sh b/tools/venv3.sh similarity index 100% rename from bootstrap/dev/venv3.sh rename to tools/venv3.sh