Merge branch 'master' into deprecation-warning

README.rst

@@ -3,9 +3,9 @@
 Disclaimer
 ==========
 
-The Let's Encrypt Client is **BETA SOFTWARE**. It contains plenty of bugs and
-rough edges, and should be tested thoroughly in staging environments before use
-on production systems.
+Certbot (previously, the Let's Encrypt client) is **BETA SOFTWARE**. It
+contains plenty of bugs and rough edges, and should be tested thoroughly in
+staging environments before use on production systems.
 
 For more information regarding the status of the project, please see
 https://letsencrypt.org. Be sure to checkout the
@@ -128,16 +128,15 @@ System Requirements
 ===================
 
 The Let's Encrypt Client presently only runs on Unix-ish OSes that include
-Python 2.6 or 2.7; Python 3.x support will be added after the Public Beta
-launch. The client requires root access in order to write to
-``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to
-bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and
-modify webserver configurations (if you use the ``apache`` or ``nginx``
-plugins).  If none of these apply to you, it is theoretically possible to run
-without root privileges, but for most users who want to avoid running an ACME
-client as root, either `letsencrypt-nosudo
-<https://github.com/diafygi/letsencrypt-nosudo>`_ or `simp_le
-<https://github.com/kuba/simp_le>`_ are more appropriate choices.
+Python 2.6 or 2.7; Python 3.x support will hopefully be added in the future. The
+client requires root access in order to write to ``/etc/letsencrypt``,
+``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443
+(if you use the ``standalone`` plugin) and to read and modify webserver
+configurations (if you use the ``apache`` or ``nginx`` plugins).  If none of
+these apply to you, it is theoretically possible to run without root privileges,
+but for most users who want to avoid running an ACME client as root, either
+`letsencrypt-nosudo <https://github.com/diafygi/letsencrypt-nosudo>`_ or
+`simp_le <https://github.com/kuba/simp_le>`_ are more appropriate choices.
 
 The Apache plugin currently requires a Debian-based OS with augeas version
 1.0; this includes Ubuntu 12.04+ and Debian 7+.

certbot-apache/certbot_apache/augeas_lens/httpd.aug

@@ -45,8 +45,8 @@ autoload xfm
 let dels (s:string)     = del s s
 
 (* deal with continuation lines *)
-let sep_spc             = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " "
-let sep_osp             = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)/ ""
+let sep_spc             = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)+/ " "
+let sep_osp             = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)*/ ""
 let sep_eq              = del /[ \t]*=[ \t]*/ "="
 
 let nmtoken             = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/
@@ -58,8 +58,8 @@ let empty               = Util.empty_dos
 let indent              = Util.indent
 
 (* borrowed from shellvars.aug *)
-let char_arg_dir  = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/
-let char_arg_sec  = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'/
+let char_arg_dir  = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'|\\\\ /
+let char_arg_sec  = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'|\\\\ /
 let char_arg_wl   = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/
 
 let cdot = /\\\\./

certbot-apache/certbot_apache/tests/apache-conf-files/passing/escaped-space-arguments-2735.conf

@@ -0,0 +1,2 @@
+RewriteCond %{HTTP:Content-Disposition} \.php [NC]
+RewriteCond %{THE_REQUEST} ^[A-Z]{3,9}\ /.+/trackback/?\ HTTP/ [NC]

certbot-apache/certbot_apache/tests/apache-conf-files/passing/section-empty-continuations-2731.conf

@@ -0,0 +1,247 @@
+#ATTENTION!
+#
+#DO NOT MODIFY THIS FILE BECAUSE IT WAS GENERATED AUTOMATICALLY,
+#SO ALL YOUR CHANGES WILL BE LOST THE NEXT TIME THE FILE IS GENERATED.
+
+NameVirtualHost 192.168.100.218:80
+NameVirtualHost 10.128.178.192:80
+
+NameVirtualHost 192.168.100.218:443
+NameVirtualHost 10.128.178.192:443
+
+
+ServerName "254020-web1.example.com"
+ServerAdmin "name@example.com"
+
+DocumentRoot "/tmp"
+
+<IfModule mod_logio.c>
+    LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog
+</IfModule>
+<IfModule !mod_logio.c>
+    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" plesklog
+</IfModule>
+
+	TraceEnable off
+
+ServerTokens ProductOnly
+
+<Directory "/var/www/vhosts">
+    AllowOverride "All"
+    Options SymLinksIfOwnerMatch
+    Order allow,deny
+    Allow from all
+
+<IfModule sapi_apache2.c>
+php_admin_flag engine off
+</IfModule>
+
+<IfModule mod_php5.c>
+php_admin_flag engine off
+</IfModule>
+
+</Directory>
+
+<Directory "/usr/lib/mailman">
+    AllowOverride All
+    Options SymLinksIfOwnerMatch
+    Order allow,deny
+    Allow from all
+    <IfModule sapi_apache2.c>
+        php_admin_flag engine off
+    </IfModule>
+    <IfModule mod_php5.c>
+        php_admin_flag engine off
+    </IfModule>
+</Directory>
+
+<IfModule mod_headers.c>
+    Header add X-Powered-By PleskLin
+</IfModule>
+
+<IfModule mod_jk.c>
+   JkWorkersFile "/etc/httpd/conf/workers.properties"
+      JkLogFile /var/log/httpd/mod_jk.log
+   JkLogLevel info
+</IfModule>
+
+#Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf"
+
+
+    <VirtualHost \
+        192.168.100.218:80 \
+        10.128.178.192:80 \
+         \
+    >
+     	ServerName "default"
+        UseCanonicalName Off
+        DocumentRoot "/tmp"
+        ScriptAlias /cgi-bin/ "/var/www/vhosts/default/cgi-bin"
+
+
+        <IfModule mod_ssl.c>
+            SSLEngine off
+        </IfModule>
+
+        <Directory "/var/www/vhosts/default/cgi-bin">
+            AllowOverride None
+            Options None
+            Order allow,deny
+            Allow from all
+        </Directory>
+
+        <Directory /var/www/vhosts/default/htdocs>
+
+<IfModule sapi_apache2.c>
+php_admin_flag engine on
+</IfModule>
+
+<IfModule mod_php5.c>
+php_admin_flag engine on
+</IfModule>
+
+	</Directory>
+
+    </VirtualHost>
+
+<IfModule mod_ssl.c>
+
+    <VirtualHost \
+        192.168.100.218:443 \
+         \
+    >
+     	ServerName "default-192_168_100_218"
+        UseCanonicalName Off
+        DocumentRoot "/tmp"
+        ScriptAlias /cgi-bin/ "/var/www/vhosts/default/cgi-bin"
+
+
+        SSLEngine on
+        SSLVerifyClient none
+        #SSLCertificateFile "/usr/local/psa/var/certificates/cert-9MgutN"
+
+        #SSLCACertificateFile "/usr/local/psa/var/certificates/cert-s6Wx3P"
+
+        <Directory "/var/www/vhosts/default/cgi-bin">
+            AllowOverride None
+            Options None
+            Order allow,deny
+            Allow from all
+        </Directory>
+
+        <Directory /var/www/vhosts/default/htdocs>
+
+<IfModule sapi_apache2.c>
+php_admin_flag engine on
+</IfModule>
+
+<IfModule mod_php5.c>
+php_admin_flag engine on
+</IfModule>
+
+	</Directory>
+
+    </VirtualHost>
+    <VirtualHost \
+        10.128.178.192:443 \
+         \
+    >
+     	ServerName "default-10_128_178_192"
+        UseCanonicalName Off
+        DocumentRoot "/tmp"
+        ScriptAlias /cgi-bin/ "/var/www/vhosts/default/cgi-bin"
+
+
+        SSLEngine on
+        SSLVerifyClient none
+        #SSLCertificateFile "/usr/local/psa/var/certificates/certxfb6025"
+
+
+        <Directory "/var/www/vhosts/default/cgi-bin">
+            AllowOverride None
+            Options None
+            Order allow,deny
+            Allow from all
+        </Directory>
+
+        <Directory /var/www/vhosts/default/htdocs>
+
+<IfModule sapi_apache2.c>
+php_admin_flag engine on
+</IfModule>
+
+<IfModule mod_php5.c>
+php_admin_flag engine on
+</IfModule>
+
+	</Directory>
+
+    </VirtualHost>
+
+</IfModule>
+
+
+<VirtualHost \
+	192.168.100.218:80 \
+                10.128.178.192:80 \
+             \
+>
+        DocumentRoot "/tmp"
+        ServerName lists
+        ServerAlias lists.*
+        UseCanonicalName Off
+
+        ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/"
+
+        Alias "/icons/" "/var/www/icons/"
+        Alias "/pipermail/" "/var/lib/mailman/archives/public/"
+
+        <IfModule mod_ssl.c>
+            SSLEngine off
+        </IfModule>
+
+
+        <Directory /var/lib/mailman/archives/>
+            Options FollowSymLinks
+            Order allow,deny
+            Allow from all
+        </Directory>
+
+    </VirtualHost>
+
+<IfModule mod_ssl.c>
+<VirtualHost \
+        192.168.100.218:443 \
+                10.128.178.192:443 \
+             \
+>
+        DocumentRoot "/tmp"
+        ServerName lists
+        ServerAlias lists.*
+        UseCanonicalName Off
+
+        ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/"
+
+        Alias "/icons/" "/var/www/icons/"
+        Alias "/pipermail/" "/var/lib/mailman/archives/public/"
+
+        SSLEngine on
+        SSLVerifyClient none
+        #SSLCertificateFile "/usr/local/psa/var/certificates/certxfb6025"
+
+
+        <Directory /var/lib/mailman/archives/>
+            Options FollowSymLinks
+            Order allow,deny
+            Allow from all
+        </Directory>
+
+    </VirtualHost>
+</IfModule>
+
+<IfModule mod_rpaf.c>
+    RPAFproxy_ips  192.168.100.218 10.128.178.192
+</IfModule>
+<IfModule mod_rpaf-2.0.c>
+    RPAFproxy_ips  192.168.100.218 10.128.178.192
+</IfModule>

certbot-apache/certbot_apache/tests/tls_sni_01_test.py

@@ -4,6 +4,7 @@ import shutil
 
 import mock
 
+from certbot import errors
 from certbot.plugins import common_test
 
 from certbot_apache import obj
@@ -137,6 +138,16 @@ class TlsSniPerformTest(util.ApacheTest):
             set([obj.Addr.fromstring("*:443")]),
             self.sni._get_addrs(self.achalls[0]))
 
+    def test_get_addrs_no_vhost_found(self):
+        self.sni.configurator.choose_vhost = mock.Mock(
+            side_effect=errors.MissingCommandlineFlag(
+                "Failed to run Apache plugin non-interactively"))
+
+        # pylint: disable=protected-access
+        self.assertEqual(
+            set([obj.Addr.fromstring("*:443")]),
+            self.sni._get_addrs(self.achalls[0]))
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tls_sni_01.py

@@ -4,6 +4,7 @@ import os
 import logging
 
 from certbot.plugins import common
+from certbot.errors import PluginError, MissingCommandlineFlag
 
 from certbot_apache import obj
 from certbot_apache import parser
@@ -116,12 +117,21 @@ class ApacheTlsSni01(common.TLSSNI01):
 
     def _get_addrs(self, achall):
         """Return the Apache addresses needed for TLS-SNI-01."""
-        vhost = self.configurator.choose_vhost(achall.domain, temp=True)
         # TODO: Checkout _default_ rules.
         addrs = set()
         default_addr = obj.Addr(("*", str(
             self.configurator.config.tls_sni_01_port)))
 
+        try:
+            vhost = self.configurator.choose_vhost(achall.domain, temp=True)
+        except (PluginError, MissingCommandlineFlag):
+            # We couldn't find the virtualhost for this domain, possibly
+            # because it's a new vhost that's not configured yet (GH #677),
+            # or perhaps because there were multiple <VirtualHost> sections
+            # in the config file (GH #1042).  See also GH #2600.
+            addrs.add(default_addr)
+            return addrs
+
         for addr in vhost.addrs:
             if "_default_" == addr.get_addr():
                 addrs.add(default_addr)

certbot/cli.py

@@ -38,7 +38,7 @@ helpful_parser = None
 # fails safely
 
 LEAUTO = "letsencrypt-auto"
-fragment = os.path.join(".local", "share", "certbot")
+fragment = os.path.join(".local", "share", "letsencrypt")
 cli_command = LEAUTO if fragment in sys.argv[0] else "certbot"
 
 # Argparse's help formatting has a lot of unhelpful peculiarities, so we want

certbot/client.py

@@ -245,8 +245,9 @@ class Client(object):
                 domains,
                 self.config.allow_subset_of_names)
 
-        domains = [a.body.identifier.value.encode('ascii')
-                                          for a in authzr]
+        auth_domains = set(a.body.identifier.value.encode('ascii')
+                           for a in authzr)
+        domains = [d for d in domains if d in auth_domains]
 
         # Create CSR from names
         key = crypto_util.init_save_key(

certbot/main.py

@@ -528,7 +528,7 @@ def obtain_cert(config, plugins, lineage=None):
             notify("new certificate deployed with reload of {0} server; fullchain is {1}".format(
                    config.installer, lineage.fullchain), pause=False)
     elif action == "reinstall" and config.verb == "certonly":
-        notify("Certificate not yet due for renewal; no action taken.")
+        notify("Certificate not yet due for renewal; no action taken.", pause=False)
     _suggest_donation_if_appropriate(config, action)
 
 

certbot/plugins/standalone.py

@@ -120,6 +120,14 @@ def supported_challenges_validator(data):
 
     """
     challs = data.split(",")
+
+    # tls-sni-01 was dvsni during private beta
+    if "dvsni" in challs:
+        logger.info("Updating legacy standalone_supported_challenges value")
+        challs = [challenges.TLSSNI01.typ if chall == "dvsni" else chall
+                  for chall in challs]
+        data = ",".join(challs)
+
     unrecognized = [name for name in challs
                     if name not in challenges.Challenge.TYPES]
     if unrecognized:

certbot/plugins/standalone_test.py

@@ -85,6 +85,11 @@ class SupportedChallengesValidatorTest(unittest.TestCase):
     def test_not_subset(self):
         self.assertRaises(argparse.ArgumentTypeError, self._call, "dns")
 
+    def test_dvsni(self):
+        self.assertEqual("tls-sni-01", self._call("dvsni"))
+        self.assertEqual("http-01,tls-sni-01", self._call("http-01,dvsni"))
+        self.assertEqual("tls-sni-01,http-01", self._call("dvsni,http-01"))
+
 
 class AuthenticatorTest(unittest.TestCase):
     """Tests for certbot.plugins.standalone.Authenticator."""

certbot/storage.py

@@ -78,6 +78,10 @@ def write_renewal_config(o_filename, n_filename, target, relevant_data):
         if k not in relevant_data:
             del config["renewalparams"][k]
 
+    if "renew_before_expiry" not in config:
+        default_interval = constants.RENEWER_DEFAULTS["renew_before_expiry"]
+        config.initial_comment = ["renew_before_expiry = " + default_interval]
+
     # TODO: add human-readable comments explaining other available
     #       parameters
     logger.debug("Writing new config %s.", n_filename)

certbot/tests/client_test.py

@@ -201,7 +201,8 @@ class ClientTest(unittest.TestCase):
 
         authzr = []
 
-        for domain in domains:
+        # domain ordering should not be affected by authorization order
+        for domain in reversed(domains):
             authzr.append(
                 mock.MagicMock(
                     body=mock.MagicMock(

certbot/tests/main_test.py

@@ -0,0 +1,48 @@
+"""Tests for certbot.main."""
+import unittest
+
+
+import mock
+
+
+from certbot import cli
+from certbot import configuration
+from certbot.plugins import disco as plugins_disco
+
+
+class ObtainCertTest(unittest.TestCase):
+    """Tests for certbot.main.obtain_cert."""
+
+    def setUp(self):
+        self.get_utility_patch = mock.patch(
+            'certbot.main.zope.component.getUtility')
+        self.mock_get_utility = self.get_utility_patch.start()
+
+    def tearDown(self):
+        self.get_utility_patch.stop()
+
+    def _call(self, args):
+        plugins = plugins_disco.PluginsRegistry.find_all()
+        config = configuration.NamespaceConfig(
+            cli.prepare_and_parse_args(plugins, args))
+
+        from certbot import main
+        with mock.patch('certbot.main._init_le_client') as mock_init:
+            main.obtain_cert(config, plugins)
+
+        return mock_init()  # returns the client
+
+    @mock.patch('certbot.main._auth_from_domains')
+    def test_no_reinstall_text_pause(self, mock_auth):
+        mock_notification = self.mock_get_utility().notification
+        mock_notification.side_effect = self._assert_no_pause
+        mock_auth.return_value = (mock.ANY, 'reinstall')
+        self._call('certonly --webroot -d example.com -t'.split())
+
+    def _assert_no_pause(self, message, height=42, pause=True):
+        # pylint: disable=unused-argument
+        self.assertFalse(pause)
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

docs/using.rst

@@ -124,7 +124,7 @@ or ``--webroot-path /usr/share/nginx/html`` are two common webroot paths.
 
 If you're getting a certificate for many domains at once, the plugin
 needs to know where each domain's files are served from, which could
-potentially be a separate directory for each domain. When requested a
+potentially be a separate directory for each domain. When requesting a
 certificate for multiple domains, each domain will use the most recently
 specified ``--webroot-path``.  So, for instance,
 
@@ -184,11 +184,11 @@ be on a different computer.
 Nginx
 -----
 
-In the future, if you're running Nginx you can use this plugin to
-automatically obtain and install your certificate. The Nginx plugin
-is still experimental, however, and is not installed with
-letsencrypt-auto_. If installed, you can select this plugin on the
-command line by including ``--nginx``.
+In the future, if you're running Nginx you will hopefully be able to use this
+plugin to automatically obtain and install your certificate. The Nginx plugin is
+still experimental, however, and is not installed with letsencrypt-auto_. If
+installed, you can select this plugin on the command line by including
+``--nginx``.
 
 Third-party plugins
 -------------------
@@ -446,7 +446,13 @@ If you run Debian Stretch or Debian Sid, you can install letsencrypt packages.
 If you don't want to use the Apache plugin, you can omit the
 ``python-letsencrypt-apache`` package.
 
-Packages for Debian Jessie are coming in the next few weeks.
+Packages exist for Debian Jessie via backports. First you'll have to follow the
+instructions at http://backports.debian.org/Instructions/ to enable the Jessie backports
+repo, if you have not already done so. Then run:
+
+.. code-block:: shell
+
+   sudo apt-get install certbot python-certbot-apache -t jessie-backports
 
 **Fedora**
 

setup.py

@@ -39,7 +39,7 @@ install_requires = [
     'ConfigArgParse>=0.9.3',
     'configobj',
     'cryptography>=0.7',  # load_pem_x509_certificate
-    'parsedatetime',
+    'parsedatetime>=1.3',  # Calendar.parseDT
     'psutil>=2.1.0',  # net_connections introduced in 2.1.0
     'PyOpenSSL',
     'pyrfc3339',

tests/travis-integration.sh

@@ -6,14 +6,14 @@ set -o errexit
 
 source .tox/$TOXENV/bin/activate
 
-export LETSENCRYPT_PATH=`pwd`
+export CERTBOT_PATH=`pwd`
 
 cd $GOPATH/src/github.com/letsencrypt/boulder/
 
 # boulder's integration-test.py has code that knows to start and wait for the
 # boulder processes to start reliably and then will run the certbot
-# boulder-interation.sh on its own. The --letsencrypt flag says to run only the
+# boulder-interation.sh on its own. The --certbot flag says to run only the
 # certbot tests (instead of any other client tests it might run). We're
 # going to want to define a more robust interaction point between the boulder
 # and certbot tests, but that will be better built off of this.
-python test/integration-test.py --letsencrypt
+python test/integration-test.py --certbot

tools/_venv_common.sh

@@ -18,7 +18,8 @@ virtualenv --no-site-packages $VENV_NAME $VENV_ARGS
 # Separately install setuptools and pip to make sure following
 # invocations use latest
 pip install -U setuptools
-pip install -U pip
+# --force-reinstall used to fix broken pip installation on some systems
+pip install --force-reinstall -U pip
 pip install "$@"
 
 set +x