From e7226d28041a03a8f2d6d61ac84e765e8235a180 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 16:43:24 -0800 Subject: [PATCH 01/62] Automate testing with the apache-conf-library --- tests/apache-conf-files/hackish-apache-test | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index c6663551e..8efe65e42 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -18,11 +18,25 @@ function CleanupExit() { exit 1 } +FAILS=0 trap CleanupExit INT for f in *.conf ; do - echo testing "$f" + echo -n testing "$f"... sudo cp "$f" "$EA"/sites-available/ sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - sudo "$LEROOT"/venv/bin/letsencrypt --apache certonly -t + RESULT=`echo c | sudo "$LEROOT"/venv/bin/letsencrypt --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then + echo passed + else + echo failed + echo $RESULT + echo + echo + FAILS=`expr $FAILS + 1` + fi sudo rm /etc/apache2/sites-{enabled,available}/"$f" done +if [ "$FAILS" -ne 0 ] ; then + return 1 +fi +return 0 From 52705107ff76f1e713b9b903837e7c38c230f411 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 17:01:36 -0800 Subject: [PATCH 02/62] let the environment determine how letsencrypt is run --- tests/apache-conf-files/hackish-apache-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index 8efe65e42..e6f25b15f 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -9,6 +9,7 @@ export EA=/etc/apache2/ TESTDIR="`dirname $0`" LEROOT="`realpath \"$TESTDIR/../../\"`" cd $TESTDIR/passing +LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" function CleanupExit() { echo control c, exiting tests... @@ -24,7 +25,7 @@ for f in *.conf ; do echo -n testing "$f"... sudo cp "$f" "$EA"/sites-available/ sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - RESULT=`echo c | sudo "$LEROOT"/venv/bin/letsencrypt --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | sudo "$LETSENCRYPT" --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed else From 3c6af7094c67b37e8b815309fe0a492e232fccbf Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 17:21:47 -0800 Subject: [PATCH 03/62] Bugfix, and use --staging --- tests/apache-conf-files/hackish-apache-test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index e6f25b15f..c4df319ed 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -25,7 +25,7 @@ for f in *.conf ; do echo -n testing "$f"... sudo cp "$f" "$EA"/sites-available/ sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - RESULT=`echo c | sudo "$LETSENCRYPT" --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed else @@ -38,6 +38,6 @@ for f in *.conf ; do sudo rm /etc/apache2/sites-{enabled,available}/"$f" done if [ "$FAILS" -ne 0 ] ; then - return 1 + exit 1 fi -return 0 +exit 0 From bf764e4852dcf061e586789455afcb9785f39e3f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 18:01:49 -0800 Subject: [PATCH 04/62] Support appending to non-Debianish Apache setups --- tests/apache-conf-files/hackish-apache-test | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index c4df319ed..99fa123f7 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -23,8 +23,14 @@ FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... - sudo cp "$f" "$EA"/sites-available/ - sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo cp "$f" "$EA"/sites-available/ + sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + else + TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" + sudo cp -a "$APPEND_APACHECONF" "$TMP" + sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" + fi RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed @@ -35,7 +41,11 @@ for f in *.conf ; do echo FAILS=`expr $FAILS + 1` fi - sudo rm /etc/apache2/sites-{enabled,available}/"$f" + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo rm /etc/apache2/sites-{enabled,available}/"$f" + else + sudo mv "$TMP" "$APPEND_APACHECONF" + fi done if [ "$FAILS" -ne 0 ] ; then exit 1 From 9013fecc9cea8bfca8e29789520772c3bd96cd51 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 19:41:35 -0800 Subject: [PATCH 05/62] Prep for testfarming. --- tests/apache-conf-files/hackish-apache-test | 16 ++++++++++------ tests/apache-conf-files/passing/README.modules | 3 +-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index 99fa123f7..3d4336579 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -14,11 +14,19 @@ LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" function CleanupExit() { echo control c, exiting tests... if [ "$f" != "" ] ; then - sudo rm /etc/apache2/sites-{enabled,available}/"$f" + Cleanup fi exit 1 } +function Cleanup() { + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo rm /etc/apache2/sites-{enabled,available}/"$f" + else + sudo mv "$TMP" "$APPEND_APACHECONF" + fi +} + FAILS=0 trap CleanupExit INT for f in *.conf ; do @@ -41,11 +49,7 @@ for f in *.conf ; do echo FAILS=`expr $FAILS + 1` fi - if [ "$APPEND_APACHECONF" = "" ] ; then - sudo rm /etc/apache2/sites-{enabled,available}/"$f" - else - sudo mv "$TMP" "$APPEND_APACHECONF" - fi + Cleanup done if [ "$FAILS" -ne 0 ] ; then exit 1 diff --git a/tests/apache-conf-files/passing/README.modules b/tests/apache-conf-files/passing/README.modules index 7edbd3e84..32c3ef019 100644 --- a/tests/apache-conf-files/passing/README.modules +++ b/tests/apache-conf-files/passing/README.modules @@ -1,5 +1,4 @@ -Modules required to parse these conf files: - +# Modules required to parse these conf files: ssl rewrite macro From babb33991bffb51fd82acc75afa87ce4774168c4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 19:51:45 -0800 Subject: [PATCH 06/62] Neaten things with a Setup() function --- tests/apache-conf-files/hackish-apache-test | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index 3d4336579..b8caaadc0 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -19,6 +19,17 @@ function CleanupExit() { exit 1 } +function Setup() { + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo cp "$f" "$EA"/sites-available/ + sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + else + TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" + sudo cp -a "$APPEND_APACHECONF" "$TMP" + sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" + fi +} + function Cleanup() { if [ "$APPEND_APACHECONF" = "" ] ; then sudo rm /etc/apache2/sites-{enabled,available}/"$f" @@ -31,14 +42,7 @@ FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... - if [ "$APPEND_APACHECONF" = "" ] ; then - sudo cp "$f" "$EA"/sites-available/ - sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" - else - TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" - sudo cp -a "$APPEND_APACHECONF" "$TMP" - sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" - fi + Setup RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed From 03fdd03a878ce1451c6e766f32421e604ceb4f72 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 19:52:35 -0800 Subject: [PATCH 07/62] Experimentally try travis with the hackish-apache-test --- .travis.yml | 1 + tox.ini | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8dde06ceb..2b37c1bef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=lint - TOXENV=cover + - TOXENV=hackishapachetest # Only build pushes to the master branch, PRs, and branches beginning with diff --git a/tox.ini b/tox.ini index d1fafe20f..cffbc4e18 100644 --- a/tox.ini +++ b/tox.ini @@ -67,3 +67,10 @@ commands = pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt + +[testenv:hackishapachetest] +basepython = python2.7 +setenv = + LETSENCRYPT=letsencrypt +commands = + ./tests/apache-conf-files/hackish-apache-test From 9129dcbc8797f67d85d04fa29541b5cec446e2ce Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 20:04:27 -0800 Subject: [PATCH 08/62] Make sure there's always a domain name to prompt a question --- tests/apache-conf-files/hackish-apache-test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index b8caaadc0..27ee0fdf0 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -23,6 +23,13 @@ function Setup() { if [ "$APPEND_APACHECONF" = "" ] ; then sudo cp "$f" "$EA"/sites-available/ sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + sudo echo """ + + ServerName example.com + DocumentRoot /tmp/ + ErrorLog /tmp/error.log + CustomLog /tmp/requests.log combined +""" >> $EA/sites-available/throwaway-example.conf else TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" sudo cp -a "$APPEND_APACHECONF" "$TMP" @@ -33,6 +40,7 @@ function Setup() { function Cleanup() { if [ "$APPEND_APACHECONF" = "" ] ; then sudo rm /etc/apache2/sites-{enabled,available}/"$f" + sudo rm $EA/sites-available/throwaway-example.conf else sudo mv "$TMP" "$APPEND_APACHECONF" fi From d777e7fabad3f379c266382a36f970a13e85cf12 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 17 Dec 2015 15:45:47 -0800 Subject: [PATCH 09/62] This sort of works in tox; travis is unlikely due to sudo --- .travis.yml | 1 + .../letsencrypt_apache/tests}/apache-conf-files/NEEDED.txt | 0 .../tests}/apache-conf-files/failing/ipv6-1143.conf | 0 .../tests}/apache-conf-files/failing/ipv6-1143b.conf | 0 .../apache-conf-files/failing/missing-double-quote-1724.conf | 0 .../tests}/apache-conf-files/failing/multivhost-1093.conf | 0 .../tests}/apache-conf-files/failing/multivhost-1093b.conf | 0 .../tests}/apache-conf-files/hackish-apache-test | 2 +- .../tests}/apache-conf-files/passing/1626-1531.conf | 0 .../tests}/apache-conf-files/passing/README.modules | 0 .../tests}/apache-conf-files/passing/anarcat-1531.conf | 0 .../passing/drupal-errordocument-arg-1724.conf | 0 .../tests}/apache-conf-files/passing/drupal-htaccess-1531.conf | 0 .../tests}/apache-conf-files/passing/example-1755.conf | 0 .../tests}/apache-conf-files/passing/example-ssl.conf | 0 .../tests}/apache-conf-files/passing/example.conf | 0 .../apache-conf-files/passing/finalize-1243.apache2.conf.txt | 0 .../tests}/apache-conf-files/passing/finalize-1243.conf | 0 .../tests}/apache-conf-files/passing/missing-quote-1724.conf | 0 .../tests}/apache-conf-files/passing/modmacro-1385.conf | 0 .../tests}/apache-conf-files/passing/owncloud-1264.conf | 0 .../tests}/apache-conf-files/passing/roundcube-1222.conf | 0 .../tests}/apache-conf-files/passing/semacode-1598.conf | 0 .../apache-conf-files/passing/sslrequire-wordlist-1827.htaccess | 0 .../apache-conf-files/passing/two-blocks-one-line-1693.conf | 0 tox.ini | 2 +- 26 files changed, 3 insertions(+), 2 deletions(-) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/NEEDED.txt (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/ipv6-1143.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/ipv6-1143b.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/missing-double-quote-1724.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/multivhost-1093.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/multivhost-1093b.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/hackish-apache-test (97%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/1626-1531.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/README.modules (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/anarcat-1531.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/drupal-errordocument-arg-1724.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/drupal-htaccess-1531.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/example-1755.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/example-ssl.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/example.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/finalize-1243.apache2.conf.txt (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/finalize-1243.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/missing-quote-1724.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/modmacro-1385.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/owncloud-1264.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/roundcube-1222.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/semacode-1598.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/two-blocks-one-line-1693.conf (100%) diff --git a/.travis.yml b/.travis.yml index 2b37c1bef..dfbc09e2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: python services: - rabbitmq - mariadb + - apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml diff --git a/tests/apache-conf-files/NEEDED.txt b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/NEEDED.txt similarity index 100% rename from tests/apache-conf-files/NEEDED.txt rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/NEEDED.txt diff --git a/tests/apache-conf-files/failing/ipv6-1143.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143.conf similarity index 100% rename from tests/apache-conf-files/failing/ipv6-1143.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143.conf diff --git a/tests/apache-conf-files/failing/ipv6-1143b.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143b.conf similarity index 100% rename from tests/apache-conf-files/failing/ipv6-1143b.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143b.conf diff --git a/tests/apache-conf-files/failing/missing-double-quote-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/missing-double-quote-1724.conf similarity index 100% rename from tests/apache-conf-files/failing/missing-double-quote-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/missing-double-quote-1724.conf diff --git a/tests/apache-conf-files/failing/multivhost-1093.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093.conf similarity index 100% rename from tests/apache-conf-files/failing/multivhost-1093.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093.conf diff --git a/tests/apache-conf-files/failing/multivhost-1093b.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093b.conf similarity index 100% rename from tests/apache-conf-files/failing/multivhost-1093b.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093b.conf diff --git a/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test similarity index 97% rename from tests/apache-conf-files/hackish-apache-test rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index 27ee0fdf0..9e828bf2d 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -7,7 +7,7 @@ # assess, but it should be automated export EA=/etc/apache2/ TESTDIR="`dirname $0`" -LEROOT="`realpath \"$TESTDIR/../../\"`" +LEROOT="`realpath \"$TESTDIR/../../../../\"`" cd $TESTDIR/passing LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" diff --git a/tests/apache-conf-files/passing/1626-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/1626-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/1626-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/1626-1531.conf diff --git a/tests/apache-conf-files/passing/README.modules b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules similarity index 100% rename from tests/apache-conf-files/passing/README.modules rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules diff --git a/tests/apache-conf-files/passing/anarcat-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/anarcat-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/anarcat-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/anarcat-1531.conf diff --git a/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf similarity index 100% rename from tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf diff --git a/tests/apache-conf-files/passing/drupal-htaccess-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-htaccess-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/drupal-htaccess-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-htaccess-1531.conf diff --git a/tests/apache-conf-files/passing/example-1755.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-1755.conf similarity index 100% rename from tests/apache-conf-files/passing/example-1755.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-1755.conf diff --git a/tests/apache-conf-files/passing/example-ssl.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-ssl.conf similarity index 100% rename from tests/apache-conf-files/passing/example-ssl.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-ssl.conf diff --git a/tests/apache-conf-files/passing/example.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example.conf similarity index 100% rename from tests/apache-conf-files/passing/example.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example.conf diff --git a/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt similarity index 100% rename from tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt diff --git a/tests/apache-conf-files/passing/finalize-1243.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.conf similarity index 100% rename from tests/apache-conf-files/passing/finalize-1243.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.conf diff --git a/tests/apache-conf-files/passing/missing-quote-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/missing-quote-1724.conf similarity index 100% rename from tests/apache-conf-files/passing/missing-quote-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/missing-quote-1724.conf diff --git a/tests/apache-conf-files/passing/modmacro-1385.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/modmacro-1385.conf similarity index 100% rename from tests/apache-conf-files/passing/modmacro-1385.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/modmacro-1385.conf diff --git a/tests/apache-conf-files/passing/owncloud-1264.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/owncloud-1264.conf similarity index 100% rename from tests/apache-conf-files/passing/owncloud-1264.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/owncloud-1264.conf diff --git a/tests/apache-conf-files/passing/roundcube-1222.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/roundcube-1222.conf similarity index 100% rename from tests/apache-conf-files/passing/roundcube-1222.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/roundcube-1222.conf diff --git a/tests/apache-conf-files/passing/semacode-1598.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/semacode-1598.conf similarity index 100% rename from tests/apache-conf-files/passing/semacode-1598.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/semacode-1598.conf diff --git a/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess similarity index 100% rename from tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess diff --git a/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf similarity index 100% rename from tests/apache-conf-files/passing/two-blocks-one-line-1693.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf diff --git a/tox.ini b/tox.ini index cffbc4e18..1d2f12e6c 100644 --- a/tox.ini +++ b/tox.ini @@ -73,4 +73,4 @@ basepython = python2.7 setenv = LETSENCRYPT=letsencrypt commands = - ./tests/apache-conf-files/hackish-apache-test + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test From a819ee146d6369ab08fe022ae3f7481e50d32ed9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 17 Dec 2015 18:03:34 -0800 Subject: [PATCH 10/62] Experimentally try sudo in travis This may decontainerise us, so we might not want to merge even if it works... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dfbc09e2e..a0ca6576f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ branches: - /^test-.*$/ # container-based infrastructure -sudo: false +sudo: true addons: # make sure simplehttp simple verification works (custom /etc/hosts) From 7a16e2e2489bceb6cb88e21e0fa0d75900572250 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 00:17:22 -0800 Subject: [PATCH 11/62] Wrangle things to actually run in travis --- .travis.yml | 11 +++++++---- .../tests/apache-conf-files/hackish-apache-test | 1 + tox.ini | 5 +++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0ca6576f..b1fe7f55d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ env: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH matrix: - - TOXENV=py26 BOULDER_INTEGRATION=1 - - TOXENV=py27 BOULDER_INTEGRATION=1 - - TOXENV=lint - - TOXENV=cover - TOXENV=hackishapachetest +# - TOXENV=py26 BOULDER_INTEGRATION=1 +# - TOXENV=py27 BOULDER_INTEGRATION=1 +# - TOXENV=lint +# - TOXENV=cover # Only build pushes to the master branch, PRs, and branches beginning with @@ -60,6 +60,9 @@ addons: - openssl # For Boulder integration testing - rsyslog + # for hackishapachetest + - realpath + - apache2 install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index 9e828bf2d..664423d7a 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -51,6 +51,7 @@ trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup + echo running from $PWD RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed diff --git a/tox.ini b/tox.ini index 1d2f12e6c..1a637777d 100644 --- a/tox.ini +++ b/tox.ini @@ -69,8 +69,9 @@ commands = pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt [testenv:hackishapachetest] -basepython = python2.7 +#basepython = python2.7 setenv = - LETSENCRYPT=letsencrypt + LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/hackishapachetest/bin/letsencrypt commands = + pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test From 8d71b2d6c38aa062ef20a4cb10d7e9fb4cc345c3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 08:48:49 -0800 Subject: [PATCH 12/62] Install Apache modules in travis --- .travis.yml | 2 ++ .../tests/apache-conf-files/hackish-apache-test | 13 +++++++++++-- tox.ini | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1fe7f55d..6b0af69c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,8 @@ addons: # for hackishapachetest - realpath - apache2 + - libapache2-mod-wsgi + - sudo install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index 664423d7a..cf06b48af 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -46,13 +46,22 @@ function Cleanup() { fi } +# if our environment asks us to enable modules, do our best! +if [ "$1" = --debian-modules ] ; then + sudo apt-get install -y libapache2-mod-wsgi + + for mod in ssl rewrite macro wsgi deflate ; do + sudo a2enmod $mod + done +fi + + FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup - echo running from $PWD - RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | sudo "$LETSENCRYPT" -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed else diff --git a/tox.ini b/tox.ini index 1a637777d..abb934055 100644 --- a/tox.ini +++ b/tox.ini @@ -74,4 +74,4 @@ setenv = LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/hackishapachetest/bin/letsencrypt commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt - sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test --debian-modules From a0e902d405e8cd4bd3b584847ec8636affc818e0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 11:43:20 -0800 Subject: [PATCH 13/62] More module deps! --- .../tests/apache-conf-files/hackish-apache-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index cf06b48af..bfe71cc51 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -48,9 +48,9 @@ function Cleanup() { # if our environment asks us to enable modules, do our best! if [ "$1" = --debian-modules ] ; then - sudo apt-get install -y libapache2-mod-wsgi + sudo apt-get install -y libapache2-mod-{wsgi,macro} - for mod in ssl rewrite macro wsgi deflate ; do + for mod in ssl rewrite macro wsgi deflate userdir version ; do sudo a2enmod $mod done fi From 05e210d42ad5fe38a4b1b66a659e6dfe3d77e736 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 11:44:52 -0800 Subject: [PATCH 14/62] Also add dependencies to travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6b0af69c3..ab1931be1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ addons: - realpath - apache2 - libapache2-mod-wsgi + - libapache2-mod-macro - sudo install: "travis_retry pip install tox coveralls" From 47f7e70b764c4b5c760256659260059afee14df2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 21 Dec 2015 13:49:46 -0800 Subject: [PATCH 15/62] This is a more correct test --- .../tests/apache-conf-files/hackish-apache-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index bfe71cc51..c661e6435 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -62,7 +62,7 @@ for f in *.conf ; do echo -n testing "$f"... Setup RESULT=`echo c | sudo "$LETSENCRYPT" -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` - if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then + if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then echo passed else echo failed From e41ddd2cc7210563506bb5107841cfb76c27c6e2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 15:50:48 -0800 Subject: [PATCH 16/62] Rename hackishapachetest -> apacheconftest Reenable other travis tests as well as this one --- .travis.yml | 10 +++++----- .../{hackish-apache-test => apache-conf-test} | 0 tox.ini | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) rename letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/{hackish-apache-test => apache-conf-test} (100%) diff --git a/.travis.yml b/.travis.yml index ab1931be1..9efb95165 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ env: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH matrix: - - TOXENV=hackishapachetest -# - TOXENV=py26 BOULDER_INTEGRATION=1 -# - TOXENV=py27 BOULDER_INTEGRATION=1 -# - TOXENV=lint -# - TOXENV=cover + - TOXENV=py26 BOULDER_INTEGRATION=1 + - TOXENV=py27 BOULDER_INTEGRATION=1 + - TOXENV=lint + - TOXENV=cover + - TOXENV=apacheconftest # Only build pushes to the master branch, PRs, and branches beginning with diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test similarity index 100% rename from letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test diff --git a/tox.ini b/tox.ini index abb934055..aac7c15eb 100644 --- a/tox.ini +++ b/tox.ini @@ -68,10 +68,10 @@ commands = pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt -[testenv:hackishapachetest] +[testenv:apacheconftest] #basepython = python2.7 setenv = - LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/hackishapachetest/bin/letsencrypt + LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/apacheconftest/bin/letsencrypt commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt - sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test --debian-modules + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test --debian-modules From ef1973ae2888232a5b19d2c5dae4783506ccf83c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 15:58:09 -0800 Subject: [PATCH 17/62] Move recently included tests to the new apache-conf-test location --- .../tests}/apache-conf-files/passing/graphite-quote-1934.conf | 0 .../tests}/apache-conf-files/passing/rewrite-quote-1960.conf | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/graphite-quote-1934.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/rewrite-quote-1960.conf (100%) diff --git a/tests/apache-conf-files/passing/graphite-quote-1934.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf similarity index 100% rename from tests/apache-conf-files/passing/graphite-quote-1934.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf diff --git a/tests/apache-conf-files/passing/rewrite-quote-1960.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf similarity index 100% rename from tests/apache-conf-files/passing/rewrite-quote-1960.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf From d9ea151fbb2c54e5dc390b23590857de7f02cf73 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 16:04:30 -0800 Subject: [PATCH 18/62] Tweak the graphite config file so that apacheconftest passes again (At least on debian and other systems that have a www-data user...) --- .../tests/apache-conf-files/passing/graphite-quote-1934.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf index 2a8734b43..f257dd9a8 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf @@ -1,6 +1,6 @@ - WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=_graphite group=_graphite + WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=www-data group=www-data WSGIProcessGroup _graphite WSGIImportScript /usr/share/graphite-web/graphite.wsgi process-group=_graphite application-group=%{GLOBAL} WSGIScriptAlias / /usr/share/graphite-web/graphite.wsgi From 068504ddbf2d4943d4254e8bf0a8904cd347f7cd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 17:19:01 -0800 Subject: [PATCH 19/62] Correct comment --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9efb95165..4e0849e3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ addons: - openssl # For Boulder integration testing - rsyslog - # for hackishapachetest + # for apacheconftest - realpath - apache2 - libapache2-mod-wsgi From 6db40626196c790af20e5834f5b85432dd358cb8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 10:45:08 -0800 Subject: [PATCH 20/62] Split module installation into substeps - This may prevent failures if one thing is uninstallable --- .../tests/apache-conf-files/apache-conf-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index c661e6435..38d5f05c7 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -48,7 +48,8 @@ function Cleanup() { # if our environment asks us to enable modules, do our best! if [ "$1" = --debian-modules ] ; then - sudo apt-get install -y libapache2-mod-{wsgi,macro} + sudo apt-get install -y libapache2-mod-wsgi + sudo apt-get install -y libapache2-mod-macro for mod in ssl rewrite macro wsgi deflate userdir version ; do sudo a2enmod $mod From 494e6e77110c6af028df31ad270bfa70d1658ad2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 10:53:13 -0800 Subject: [PATCH 21/62] Remove TODO that has been done --- .../letsencrypt_apache/tests/apache-conf-files/apache-conf-test | 2 -- 1 file changed, 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index 38d5f05c7..ec0041534 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -3,8 +3,6 @@ # A hackish script to see if the client is behaving as expected # with each of the "passing" conf files. -# TODO presently this requires interaction and human judgement to -# assess, but it should be automated export EA=/etc/apache2/ TESTDIR="`dirname $0`" LEROOT="`realpath \"$TESTDIR/../../../../\"`" From 3dc3df4b345ee45d3b5d28a71fb1379d2e064446 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 10:58:28 -0800 Subject: [PATCH 22/62] Document the inclusion of apacheconftest in tox --- docs/contributing.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c71aefeec..6c70830b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -65,8 +65,14 @@ Testing The following tools are there to help you: -- ``tox`` starts a full set of tests. Please make sure you run it - before submitting a new pull request. +- ``tox`` starts a full set of tests. Please note that it includes + apacheconftest, which uses the system's Apache install to test config file + parsing, so it should only be run on systems that have an + experimental, non-production Apache2 install on them. ``tox -e + apacheconftest`` can be used to run those specific Apache conf tests. + +- ``tox -e py27``, ``tox -e py26`` etc, run unit tests for specific Python + versions. - ``tox -e cover`` checks the test coverage only. Calling the ``./tox.cover.sh`` script directly (or even ``./tox.cover.sh $pkg1 From ce9e3c1f94213d1c2158664436505e22e732e216 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 15:00:07 -0800 Subject: [PATCH 23/62] Some OSes don't enable the mime module by default? --- .../letsencrypt_apache/tests/apache-conf-files/apache-conf-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index ec0041534..4e0443bb7 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -49,7 +49,7 @@ if [ "$1" = --debian-modules ] ; then sudo apt-get install -y libapache2-mod-wsgi sudo apt-get install -y libapache2-mod-macro - for mod in ssl rewrite macro wsgi deflate userdir version ; do + for mod in ssl rewrite macro wsgi deflate userdir version mime ; do sudo a2enmod $mod done fi From c271dc58ce4efb884666217a182d2785a9a4bf42 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 24 Dec 2015 20:55:44 -0800 Subject: [PATCH 24/62] Fix the letsencrypt-auto update script --- tests/letstest/scripts/test_leauto_upgrades.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 70f8a2293..b7849755a 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -7,7 +7,9 @@ cd letsencrypt #git checkout v0.1.0 use --branch instead SAVE="$PIP_EXTRA_INDEX_URL" unset PIP_EXTRA_INDEX_URL +export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" ./letsencrypt-auto -v --debug --version +unset PIP_INDEX_URL export PIP_EXTRA_INDEX_URL="$SAVE" From dc5c0933df775a49d93e0401cfc7306127fcb733 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 24 Dec 2015 21:39:28 -0800 Subject: [PATCH 25/62] Definitely need --debug for this --- tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh index 476ad8bde..234e70f68 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh @@ -4,4 +4,4 @@ cd letsencrypt # help installs virtualenv and does nothing else -./letsencrypt-auto -v --help all +./letsencrypt-auto -v --debug --help all From e0837ad41f1e2bee0df17ccb77948abc1767f0c9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 25 Dec 2015 02:39:05 -0800 Subject: [PATCH 26/62] [letstest] create & reuse a persistent boulder server Reduces minimum multitester.py runtime to just a bit under 10 minutes --- tests/letstest/multitester.py | 87 ++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 5aca79b7a..60be10447 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -77,6 +77,12 @@ parser.add_argument('--saveinstances', parser.add_argument('--alt_pip', default='', help="server from which to pull candidate release packages") +parser.add_argument('--killboulder', + action='store_true', + help="do not leave a persistent boulder server running") +parser.add_argument('--boulderonly', + action='store_true', + help="only make a boulder server") cl_args = parser.parse_args() # Credential Variables @@ -292,6 +298,29 @@ def grab_letsencrypt_log(): sudo('if [ -f ./letsencrypt.log ]; then \ cat ./letsencrypt.log; else echo "[nolocallog]"; fi') +def create_client_instances(targetlist): + "Create a fleet of client instances" + instances = [] + print("Creating instances: ", end="") + for target in targetlist: + if target['virt'] == 'hvm': + machine_type = 't2.micro' + else: + machine_type = 't1.micro' + if 'userdata' in target.keys(): + userdata = target['userdata'] + else: + userdata = '' + name = 'le-%s'%target['name'] + print(name, end=" ") + instances.append(make_instance(name, + target['ami'], + KEYNAME, + machine_type=machine_type, + userdata=userdata)) + print() + return instances + #------------------------------------------------------------------------------- # SCRIPT BEGINS #------------------------------------------------------------------------------- @@ -352,30 +381,28 @@ if not sg_exists: make_security_group() time.sleep(30) +boulder_preexists = False +boulder_servers = EC2.instances.filter(Filters=[ + {'Name': 'tag:Name', 'Values': ['le-boulderserver']}, + {'Name': 'instance-state-name', 'Values': ['running']}]) + +boulder_server = next(iter(boulder_servers), None) + print("Requesting Instances...") -boulder_server = make_instance('le-boulderserver', - BOULDER_AMI, - KEYNAME, - #machine_type='t2.micro', - machine_type='t2.medium', - security_groups=['letsencrypt_test']) - -instances = [] -for target in targetlist: - if target['virt'] == 'hvm': - machine_type = 't2.micro' - else: - machine_type = 't1.micro' - if 'userdata' in target.keys(): - userdata = target['userdata'] - else: - userdata = '' - instances.append(make_instance('le-%s'%target['name'], - target['ami'], +if boulder_server: + print("Found existing boulder server:", boulder_server) + boulder_preexists = True +else: + print("Can't find a boulder server, starting one...") + boulder_server = make_instance('le-boulderserver', + BOULDER_AMI, KEYNAME, - machine_type=machine_type, - userdata=userdata)) + machine_type='t2.micro', + #machine_type='t2.medium', + security_groups=['letsencrypt_test']) +if not cl_args.boulderonly: + instances = create_client_instances(targetlist) # Configure and launch boulder server #------------------------------------------------------------------------------- @@ -383,21 +410,24 @@ print("Waiting on Boulder Server") boulder_server = block_until_instance_ready(boulder_server) print(" server %s"%boulder_server) -print("Configuring and Launching Boulder") # env.host_string defines the ssh user and host for connection env.host_string = "ubuntu@%s"%boulder_server.public_ip_address print("Boulder Server at (SSH):", env.host_string) -config_and_launch_boulder(boulder_server) -# blocking often unnecessary, but cheap EC2 VMs can get very slow -block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, - timeout=500) +if not boulder_preexists: + print("Configuring and Launching Boulder") + config_and_launch_boulder(boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) print("Boulder Server at (EC2 private ip): %s"%boulder_url) +if cl_args.boulderonly: + sys.exit(0) + # Install and launch client scripts in parallel #------------------------------------------------------------------------------- print("Uploading and running test script in parallel: %s"%cl_args.test_script) @@ -480,7 +510,8 @@ results_file.close() if not cl_args.saveinstances: print('Logs in ', LOGDIR) print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') - boulder_server.terminate() + if cl_args.killboulder: + boulder_server.terminate() terminate_and_clean(instances) else: # print login information for the boxes for debugging From f5a0268f172433f5cf91c11fc7dd8c12f5f3e3f0 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Tue, 24 Nov 2015 13:25:28 +0200 Subject: [PATCH 27/62] Adding Python 2.6/2.7 note to the docs --- docs/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index c71aefeec..ea9a9a16c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -365,10 +365,12 @@ are provided here mainly for the :ref:`developers ` reference. In general: * ``sudo`` is required as a suggested way of running privileged process +* `Python`_ 2.6/2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies +.. _Python: https://wiki.python.org/moin/BeginnersGuide/Download .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io From 3e1bc19e0f5e6ef417328c39c027c8e61ac50e2e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 25 Dec 2015 10:43:52 -0800 Subject: [PATCH 28/62] Optional flag for faster AMIs --- tests/letstest/multitester.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 60be10447..dee6968c3 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -83,6 +83,9 @@ parser.add_argument('--killboulder', parser.add_argument('--boulderonly', action='store_true', help="only make a boulder server") +parser.add_argument('--fast', + action='store_true', + help="use larger instance types to run faster (saves about a minute, probably not worth it)") cl_args = parser.parse_args() # Credential Variables @@ -304,9 +307,10 @@ def create_client_instances(targetlist): print("Creating instances: ", end="") for target in targetlist: if target['virt'] == 'hvm': - machine_type = 't2.micro' + machine_type = 't2.medium' if cl_args.fast else 't2.micro' else: - machine_type = 't1.micro' + # 32 bit systems + machine_type = 'c1.medium' if cl_args.fast else 't1.micro' if 'userdata' in target.keys(): userdata = target['userdata'] else: From f20cd73e8dc1e16ecff9d1161ec73076c05f836d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 26 Dec 2015 20:13:45 -0800 Subject: [PATCH 29/62] For now, disable apacheconftest in travis :( --- .travis.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e0849e3b..a5d6d8a85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: python services: - rabbitmq - mariadb - - apache2 + # apacheconftest + #- apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml @@ -23,7 +24,9 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=lint - TOXENV=cover - - TOXENV=apacheconftest +# Disabled for now due to requiring sudo -> causing more boulder integration +# DNS timeouts :( +# - TOXENV=apacheconftest # Only build pushes to the master branch, PRs, and branches beginning with @@ -35,7 +38,7 @@ branches: - /^test-.*$/ # container-based infrastructure -sudo: true +sudo: false addons: # make sure simplehttp simple verification works (custom /etc/hosts) @@ -61,11 +64,11 @@ addons: # For Boulder integration testing - rsyslog # for apacheconftest - - realpath - - apache2 - - libapache2-mod-wsgi - - libapache2-mod-macro - - sudo + #- realpath + #- apache2 + #- libapache2-mod-wsgi + #- libapache2-mod-macro + #- sudo install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' From 83812dc16a081ddc43d8f4204d61f0b7e73e8768 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 12:56:44 +0200 Subject: [PATCH 30/62] Abstract config file matching to os based constants --- letsencrypt-apache/letsencrypt_apache/constants.py | 3 +++ letsencrypt-apache/letsencrypt_apache/parser.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index f8712c247..800959463 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -6,6 +6,7 @@ from letsencrypt import le_util CLI_DEFAULTS_DEBIAN = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/sites-available", + vhost_files="*", ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], @@ -19,6 +20,7 @@ CLI_DEFAULTS_DEBIAN = dict( CLI_DEFAULTS_CENTOS = dict( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", + vhost_files="*.conf", ctl="apachectl", version_cmd=['apachectl', '-v'], define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], @@ -32,6 +34,7 @@ CLI_DEFAULTS_CENTOS = dict( CLI_DEFAULTS_GENTOO = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", + vhost_files="*.conf", ctl="apache2ctl", version_cmd=['/usr/sbin/apache2', '-v'], define_cmd=['/usr/sbin/apache2', '-t', '-D', 'DUMP_RUN_CFG'], diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 12704c859..5f84e9f52 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -60,9 +60,10 @@ class ApacheParser(object): self.loc.update(self._set_locations()) # Must also attempt to parse virtual host root - self._parse_file(self.vhostroot + "/*.conf") + self._parse_file(self.vhostroot + "/" + + constants.os_constant("vhost_files")) - #check to see if there were unparsed define statements + # check to see if there were unparsed define statements if version < (2, 4): if self.find_dir("Define", exclude=False): raise errors.PluginError("Error parsing runtime variables") From e3358bb15346cbcedd229c37d072c05f2198cf4a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 13:47:14 +0200 Subject: [PATCH 31/62] Abstract the remaining commands to configurable ones --- .../letsencrypt_apache/configurator.py | 22 ++++++++----------- .../letsencrypt_apache/constants.py | 9 +++++--- .../letsencrypt_apache/parser.py | 16 ++++++++------ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 6c6685257..5e2156edc 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -86,10 +86,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @classmethod def add_parser_arguments(cls, add): - add("ctl", default=constants.os_constant("ctl"), - help="Path to the 'apache2ctl' binary, used for 'configtest', " - "retrieving the Apache2 version number, and initialization " - "parameters.") add("enmod", default=constants.os_constant("enmod"), help="Path to the Apache 'a2enmod' binary.") add("dismod", default=constants.os_constant("dismod"), @@ -148,10 +144,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Verify Apache is installed - for exe in (self.conf("ctl"), self.conf("enmod"), self.conf("dismod")): - if exe is not None: - if not le_util.exe_exists(exe): - raise errors.NoInstallationError + for exe in constants.os_constant("restart_cmd")[0]: + if not le_util.exe_exists(exe): + raise errors.NoInstallationError # Make sure configuration is valid self.config_test() @@ -165,7 +160,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser = parser.ApacheParser( self.aug, self.conf("server-root"), self.conf("vhost-root"), - self.conf("ctl"), self.version) + self.version) # Check for errors in parsing files with Augeas self.check_parsing_errors("httpd.aug") @@ -1277,7 +1272,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Modules can enable additional config files. Variables may be defined # within these new configuration sections. # Reload is not necessary as DUMP_RUN_CFG uses latest config. - self.parser.update_runtime_variables(self.conf("ctl")) + self.parser.update_runtime_variables() def _add_parser_mod(self, mod_name): """Shortcut for updating parser modules.""" @@ -1315,7 +1310,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script([self.conf("ctl"), "graceful"]) + le_util.run_script(constants.os_constant("restart_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1326,7 +1321,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script([self.conf("ctl"), "configtest"]) + le_util.run_script(constants.os_constant("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1346,7 +1341,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): constants.os_constant("version_cmd")) except errors.SubprocessError: raise errors.PluginError( - "Unable to run %s -v" % self.conf("ctl")) + "Unable to run %s -v" % + constants.os_constant("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index 800959463..8ac88b197 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -7,9 +7,10 @@ CLI_DEFAULTS_DEBIAN = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/sites-available", vhost_files="*", - ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", @@ -21,9 +22,10 @@ CLI_DEFAULTS_CENTOS = dict( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", - ctl="apachectl", version_cmd=['apachectl', '-v'], define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", @@ -35,9 +37,10 @@ CLI_DEFAULTS_GENTOO = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - ctl="apache2ctl", version_cmd=['/usr/sbin/apache2', '-v'], define_cmd=['/usr/sbin/apache2', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 5f84e9f52..593c807cc 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -28,7 +28,7 @@ class ApacheParser(object): arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") fnmatch_chars = set(["*", "?", "\\", "[", "]"]) - def __init__(self, aug, root, vhostroot, ctl, version=(2, 4)): + def __init__(self, aug, root, vhostroot, version=(2, 4)): # Note: Order is important here. # This uses the binary, so it can be done first. @@ -37,7 +37,7 @@ class ApacheParser(object): # This only handles invocation parameters and Define directives! self.variables = {} if version >= (2, 4): - self.update_runtime_variables(ctl) + self.update_runtime_variables() self.aug = aug # Find configuration root and make sure augeas can parse it. @@ -92,7 +92,7 @@ class ApacheParser(object): self.modules.add( os.path.basename(self.get_arg(match_filename))[:-2] + "c") - def update_runtime_variables(self, ctl): + def update_runtime_variables(self): """" .. note:: Compile time variables (apache2ctl -V) are not used within the @@ -102,7 +102,7 @@ class ApacheParser(object): .. todo:: Create separate compile time variables... simply for arg_get() """ - stdout = self._get_runtime_cfg(ctl) + stdout = self._get_runtime_cfg() variables = dict() matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) @@ -122,7 +122,7 @@ class ApacheParser(object): self.variables = variables - def _get_runtime_cfg(self, ctl): # pylint: disable=no-self-use + def _get_runtime_cfg(self): # pylint: disable=no-self-use """Get runtime configuration info. :returns: stdout from DUMP_RUN_CFG @@ -137,9 +137,11 @@ class ApacheParser(object): except (OSError, ValueError): logger.error( - "Error accessing %s for runtime parameters!%s", ctl, os.linesep) + "Error running command %s for runtime parameters!%s", + constants.os_constant("define_cmd"), os.linesep) raise errors.MisconfigurationError( - "Error accessing loaded Apache parameters: %s", ctl) + "Error accessing loaded Apache parameters: %s", + constants.os_constant("define_cmd")) # Small errors that do not impede if proc.returncode != 0: logger.warn("Error in checking parameter list: %s", stderr) From ab069741f2c4cf32b911180e1a166c809a144c0a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 14:03:50 +0200 Subject: [PATCH 32/62] Fix tests --- .../tests/constants_test.py | 9 +++++--- .../letsencrypt_apache/tests/parser_test.py | 22 ++++++++++--------- .../letsencrypt_apache/tests/util.py | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py index 63eb5c783..289b61bb1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py @@ -11,14 +11,17 @@ class ConstantsTest(unittest.TestCase): @mock.patch("letsencrypt.le_util.get_os_info") def test_get_debian_value(self, os_info): os_info.return_value = ('Debian', '', '') - self.assertEqual(constants.os_constant("ctl"), "apache2ctl") + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/apache2/sites-available") @mock.patch("letsencrypt.le_util.get_os_info") def test_get_centos_value(self, os_info): os_info.return_value = ('CentOS Linux', '', '') - self.assertEqual(constants.os_constant("ctl"), "apachectl") + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/httpd/conf.d") @mock.patch("letsencrypt.le_util.get_os_info") def test_get_default_value(self, os_info): os_info.return_value = ('Nonexistent Linux', '', '') - self.assertEqual(constants.os_constant("ctl"), "apache2ctl") + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/apache2/sites-available") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 023b3990a..b871f89b7 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -145,24 +145,26 @@ class BasicParserTest(util.ParserTest): expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.update_runtime_variables("ctl") + self.parser.update_runtime_variables() self.assertEqual(self.parser.variables, expected_vars) @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_vars_bad_output(self, mock_cfg): mock_cfg.return_value = "Define: TLS=443=24" - self.parser.update_runtime_variables("ctl") + self.parser.update_runtime_variables() mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24" self.assertRaises( - errors.PluginError, self.parser.update_runtime_variables, "ctl") + errors.PluginError, self.parser.update_runtime_variables) + @mock.patch("letsencrypt_apache.constants.os_constant") @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen): + def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): mock_popen.side_effect = OSError + mock_const.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, - self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables) @mock.patch("letsencrypt_apache.parser.subprocess.Popen") def test_update_runtime_vars_bad_exit(self, mock_popen): @@ -170,7 +172,7 @@ class BasicParserTest(util.ParserTest): mock_popen.returncode = -1 self.assertRaises( errors.MisconfigurationError, - self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables) class ParserInitTest(util.ApacheTest): @@ -191,7 +193,7 @@ class ParserInitTest(util.ApacheTest): self.assertRaises( errors.PluginError, ApacheParser, self.aug, os.path.relpath(self.config_path), - "/dummy/vhostpath", "ctl", version=(2, 2, 22)) + "/dummy/vhostpath", version=(2, 2, 22)) def test_root_normalized(self): from letsencrypt_apache.parser import ApacheParser @@ -203,7 +205,7 @@ class ParserInitTest(util.ApacheTest): "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2") parser = ApacheParser(self.aug, path, - "/dummy/vhostpath", "dummy_ctl") + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) @@ -213,7 +215,7 @@ class ParserInitTest(util.ApacheTest): "update_runtime_variables"): parser = ApacheParser( self.aug, os.path.relpath(self.config_path), - "/dummy/vhostpath", "dummy_ctl") + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) @@ -223,7 +225,7 @@ class ParserInitTest(util.ApacheTest): "update_runtime_variables"): parser = ApacheParser( self.aug, self.config_path + os.path.sep, - "/dummy/vhostpath", "dummy_ctl") + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 95c95e6a9..798d4814b 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -58,7 +58,7 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): self.parser = ApacheParser( - self.aug, self.config_path, self.vhost_path, "dummy_ctl_path") + self.aug, self.config_path, self.vhost_path) def get_apache_configurator( From 3fadfb5444b38e65fb60507c2592eaeac5cdde6f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 15:56:24 +0200 Subject: [PATCH 33/62] Fixed the find exe condition --- letsencrypt-apache/letsencrypt_apache/configurator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 5e2156edc..836d77135 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -144,9 +144,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Verify Apache is installed - for exe in constants.os_constant("restart_cmd")[0]: - if not le_util.exe_exists(exe): - raise errors.NoInstallationError + if not le_util.exe_exists(constants.os_constant("restart_cmd")[0]): + raise errors.NoInstallationError # Make sure configuration is valid self.config_test() From fd4f6fb2eef3fd24d427836023918103bac08ada Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 29 Dec 2015 08:47:14 +0000 Subject: [PATCH 34/62] Use GH pages for IETF spec repo link --- acme/acme/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 0f5f0e4bd..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -3,10 +3,10 @@ This module is an implementation of the `ACME protocol`_. Latest supported version: `draft-ietf-acme-01`_. -.. _`ACME protocol`: https://github.com/ietf-wg-acme/acme/ + +.. _`ACME protocol`: https://ietf-wg-acme.github.io/acme .. _`draft-ietf-acme-01`: https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 - """ From 7788799a9bdb6a67f25833ccba031799b7b6429a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 29 Dec 2015 08:55:13 +0000 Subject: [PATCH 35/62] Staging URI in dev-cli.ini example --- examples/cli.ini | 3 --- examples/dev-cli.ini | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cli.ini b/examples/cli.ini index 6b6b05d7d..f0c993c57 100644 --- a/examples/cli.ini +++ b/examples/cli.ini @@ -5,9 +5,6 @@ # Use a 4096 bit RSA key instead of 2048 rsa-key-size = 4096 -# Always use the staging/testing server -server = https://acme-staging.api.letsencrypt.org/directory - # Uncomment and update to register with the specified e-mail address # email = foo@example.com diff --git a/examples/dev-cli.ini b/examples/dev-cli.ini index be703814a..c02038ca1 100644 --- a/examples/dev-cli.ini +++ b/examples/dev-cli.ini @@ -1,3 +1,6 @@ +# Always use the staging/testing server - avoids rate limiting +server = https://acme-staging.api.letsencrypt.org/directory + # This is an example configuration file for developers config-dir = /tmp/le/conf work-dir = /tmp/le/conf From 8f984bd24f2779490bf526d47f7fbe14633fcfb1 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 1 Jan 2016 16:35:57 -0800 Subject: [PATCH 36/62] Better Nginx error handling. Raise MisconfigurationError on restart failure, so we don't attempt to continue with an authorization we know will fail. Log at debug level the config files that are about to be written out, so it's easier to debug restart failures. Fix https://github.com/letsencrypt/letsencrypt/issues/942: Error out if adding a conflicting directive. Remove unnecessarily-inserted access_log and error_log directives. These were added to make integration testing easier but are no longer needed. Incidentally this makes the plugin work with some configs where it wouldn't have worked previously. Change the semantics of add_server_directives with replace=True so only the first instance of a given directive is replaced, not all of them. This works fine with the one place in the code that calls add_server_directives with replace=True, because all of the involved directives aren't allowed to be duplicated in a given block. Make add_http_directives do inserts into the structure itself, since its needs were significantly different than the more general add_server_directives. This also allows us to narrow the scope of the `block.insert(0, directive)` hack that we inserted to work around https://trac.nginx.org/nginx/ticket/810, since it's only necessary for http blocks. --- .../letsencrypt_nginx/configurator.py | 24 ++---- letsencrypt-nginx/letsencrypt_nginx/parser.py | 81 +++++++++++++------ .../tests/configurator_test.py | 63 ++++++++------- .../letsencrypt_nginx/tests/parser_test.py | 29 ++++--- .../tests/boulder-integration.conf.sh | 6 +- 5 files changed, 115 insertions(+), 88 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index aaaf43c5f..89b1145e7 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -311,17 +311,11 @@ class NginxConfigurator(common.Plugin): """ snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() ssl_block = [['listen', '{0} ssl'.format(self.config.tls_sni_01_port)], - # access and error logs necessary for integration - # testing (non-root) - ['access_log', os.path.join( - self.config.work_dir, 'access.log')], - ['error_log', os.path.join( - self.config.work_dir, 'error.log')], ['ssl_certificate', snakeoil_cert], ['ssl_certificate_key', snakeoil_key], ['include', self.parser.loc["ssl_options"]]] self.parser.add_server_directives( - vhost.filep, vhost.names, ssl_block) + vhost.filep, vhost.names, ssl_block, replace=False) vhost.ssl = True vhost.raw.extend(ssl_block) vhost.addrs.add(obj.Addr( @@ -384,7 +378,7 @@ class NginxConfigurator(common.Plugin): [['return', '301 https://$host$request_uri']] ]] self.parser.add_server_directives( - vhost.filep, vhost.names, redirect_block) + vhost.filep, vhost.names, redirect_block, replace=False) logger.info("Redirecting all traffic to ssl in %s", vhost.filep) ###################################### @@ -393,11 +387,10 @@ class NginxConfigurator(common.Plugin): def restart(self): """Restarts nginx server. - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If either the reload fails. """ - return nginx_restart(self.conf('ctl'), self.nginx_conf) + nginx_restart(self.conf('ctl'), self.nginx_conf) def config_test(self): # pylint: disable=no-self-use """Check the configuration of Nginx for errors. @@ -631,19 +624,16 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): if nginx_proc.returncode != 0: # Enter recovery routine... - logger.error("Nginx Restart Failed!\n%s\n%s", stdout, stderr) - return False + raise errors.MisconfigurationError( + "nginx restart failed:\n%s\n%s" % (stdout, stderr)) except (OSError, ValueError): - logger.fatal("Nginx Restart Failed - Please Check the Configuration") - sys.exit(1) + raise errors.MisconfigurationError("nginx restart failed") # Nginx can take a moment to recognize a newly added TLS SNI servername, so sleep # for a second. TODO: Check for expected servername and loop until it # appears or return an error if looping too long. time.sleep(1) - return True - def temp_install(options_ssl): """Temporary install for convenience.""" diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index 14db2f8b7..1d424f834 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -213,6 +213,7 @@ class NginxParser(object): if ext: filename = filename + os.path.extsep + ext try: + logger.debug('Dumping to %s:\n%s', filename, nginxparser.dumps(tree)) with open(filename, 'w') as _file: nginxparser.dump(tree, _file) except IOError: @@ -252,7 +253,7 @@ class NginxParser(object): return server_names == names def add_server_directives(self, filename, names, directives, - replace=False): + replace): """Add or replace directives in the first server block with names. ..note :: If replace is True, this raises a misconfiguration error @@ -269,20 +270,27 @@ class NginxParser(object): :param bool replace: Whether to only replace existing directives """ - _do_for_subarray(self.parsed[filename], - lambda x: self._has_server_names(x, names), - lambda x: _add_directives(x, directives, replace)) + try: + _do_for_subarray(self.parsed[filename], + lambda x: self._has_server_names(x, names), + lambda x: _add_directives(x, directives, replace)) + except errors.MisconfigurationError as err: + raise errors.MisconfigurationError("Problem in %s: %s" % (filename, err.message)) def add_http_directives(self, filename, directives): """Adds directives to the first encountered HTTP block in filename. + We insert new directives at the top of the block to work around + https://trac.nginx.org/nginx/ticket/810: If the first server block + doesn't enable OCSP stapling, stapling is broken for all blocks. + :param str filename: The absolute filename of the config file :param list directives: The directives to add """ _do_for_subarray(self.parsed[filename], lambda x: x[0] == ['http'], - lambda x: _add_directives(x[1], [directives], False)) + lambda x: x[1].insert(0, directives)) def get_all_certs_keys(self): """Gets all certs and keys in the nginx config. @@ -467,9 +475,14 @@ def _parse_server(server): return parsed_server -def _add_directives(block, directives, replace=False): - """Adds or replaces directives in a block. If the directive doesn't exist in - the entry already, raises a misconfiguration error. +def _add_directives(block, directives, replace): + """Adds or replaces directives in a config block. + + When replace=False, it's an error to try and add a directive that already + exists in the config block with a conflicting value. + + When replace=True, a directive with the same name MUST already exist in the + config block, and the first instance will be replaced. ..todo :: Find directives that are in included files. @@ -478,21 +491,39 @@ def _add_directives(block, directives, replace=False): """ for directive in directives: - if not replace: - # We insert new directives at the top of the block, mostly - # to work around https://trac.nginx.org/nginx/ticket/810 - # Only add directive if its not already in the block - if directive not in block: - block.insert(0, directive) - else: - changed = False - if len(directive) == 0: - continue - for index, line in enumerate(block): - if len(line) > 0 and line[0] == directive[0]: - block[index] = directive - changed = True - if not changed: + _add_directive(block, directive, replace) + +repeatable_directives = set(['server_name', 'listen', 'include']) + +def _add_directive(block, directive, replace): + """Adds or replaces a single directive in a config block. + + See _add_directives for more documentation. + + """ + location = -1 + # Find the index of a config line where the name of the directive matches + # the name of the directive we want to add. + for index, line in enumerate(block): + if len(line) > 0 and line[0] == directive[0]: + location = index + break + if replace: + if location == -1: + raise errors.MisconfigurationError( + 'expected directive for %s in the Nginx ' + 'config but did not find it.' % directive[0]) + block[location] = directive + else: + # Append directive. Fail if the name is not a repeatable directive name, + # and there is already a copy of that directive with a different value + # in the config file. + if location != -1 and directive[0].__str__() not in repeatable_directives: + if block[location][1] == directive[1]: + pass + else: raise errors.MisconfigurationError( - 'Let\'s Encrypt expected directive for %s in the Nginx ' - 'config but did not find it.' % directive[0]) + 'tried to insert directive "%s" but found conflicting "%s".' % ( + directive, block[location])) + else: + block.append(directive) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..f6a4f7eee 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -65,16 +65,19 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') self.config.parser.add_server_directives( filep, set(['.example.com', 'example.*']), - [['listen', '5001 ssl']]) + [['listen', '5001 ssl']], + replace=False) self.config.save() # pylint: disable=protected-access parsed = self.config.parser._parse_files(filep, override=True) - self.assertEqual([[['server'], [['listen', '5001 ssl'], + self.assertEqual([[['server'], [ ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*']]]], + ['server_name', 'example.*'], + ['listen', '5001 ssl'] + ]]], parsed[0]) def test_choose_vhost(self): @@ -154,38 +157,36 @@ class NginxConfiguratorTest(util.NginxTest): parsed_server_conf = util.filter_comments(self.config.parser.parsed[server_conf]) parsed_nginx_conf = util.filter_comments(self.config.parser.parsed[nginx_conf]) - access_log = os.path.join(self.work_dir, "access.log") - error_log = os.path.join(self.work_dir, "error.log") self.assertEqual([[['server'], - [['include', self.config.parser.loc["ssl_options"]], - ['ssl_certificate_key', 'example/key.pem'], - ['ssl_certificate', 'example/fullchain.pem'], - ['error_log', error_log], - ['access_log', access_log], - - ['listen', '5001 ssl'], + [ ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*']]]], + ['server_name', 'example.*'], + + ['listen', '5001 ssl'], + ['ssl_certificate', 'example/fullchain.pem'], + ['ssl_certificate_key', 'example/key.pem'], + ['include', self.config.parser.loc["ssl_options"]] + ]]], parsed_example_conf) self.assertEqual([['server_name', 'somename alias another.alias']], parsed_server_conf) - self.assertTrue(util.contains_at_depth(parsed_nginx_conf, - [['server'], - [['include', self.config.parser.loc["ssl_options"]], - ['ssl_certificate_key', '/etc/nginx/key.pem'], - ['ssl_certificate', '/etc/nginx/fullchain.pem'], - ['error_log', error_log], - ['access_log', access_log], - ['listen', '5001 ssl'], - ['listen', '8000'], - ['listen', 'somename:8080'], - ['include', 'server.conf'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html index.htm']]]]], - 2)) + self.assertTrue(util.contains_at_depth( + parsed_nginx_conf, + [['server'], + [ + ['listen', '8000'], + ['listen', 'somename:8080'], + ['include', 'server.conf'], + [['location', '/'], + [['root', 'html'], + ['index', 'index.html index.htm']]], + ['listen', '5001 ssl'], + ['ssl_certificate', '/etc/nginx/fullchain.pem'], + ['ssl_certificate_key', '/etc/nginx/key.pem'], + ['include', self.config.parser.loc["ssl_options"]]]], + 2)) def test_get_all_certs_keys(self): nginx_conf = self.config.parser.abs_path('nginx.conf') @@ -297,19 +298,19 @@ class NginxConfiguratorTest(util.NginxTest): mocked = mock_popen() mocked.communicate.return_value = ('', '') mocked.returncode = 0 - self.assertTrue(self.config.restart()) + self.config.restart() @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_nginx_restart_fail(self, mock_popen): mocked = mock_popen() mocked.communicate.return_value = ('', '') mocked.returncode = 1 - self.assertFalse(self.config.restart()) + self.assertRaises(errors.MisconfigurationError, self.config.restart) @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_no_nginx_start(self, mock_popen): mock_popen.side_effect = OSError("Can't find program") - self.assertRaises(SystemExit, self.config.restart) + self.assertRaises(errors.MisconfigurationError, self.config.restart) @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_config_test(self, mock_popen): diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 2d6156429..6559a5df6 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -127,7 +127,8 @@ class NginxParserTest(util.NginxTest): set(['localhost', r'~^(www\.)?(example|bar)\.']), [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert.pem']]) + '/etc/ssl/cert.pem']], + replace=False) ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')]) self.assertEqual(1, len(re.findall(ssl_re, dump))) @@ -136,12 +137,15 @@ class NginxParserTest(util.NginxTest): names = set(['alias', 'another.alias', 'somename']) nparser.add_server_directives(server_conf, names, [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert2.pem']]) - nparser.add_server_directives(server_conf, names, [['foo', 'bar']]) + '/etc/ssl/cert2.pem']], + replace=False) + nparser.add_server_directives(server_conf, names, [['foo', 'bar']], + replace=False) self.assertEqual(nparser.parsed[server_conf], - [['ssl_certificate', '/etc/ssl/cert2.pem'], + [['server_name', 'somename alias another.alias'], ['foo', 'bar'], - ['server_name', 'somename alias another.alias']]) + ['ssl_certificate', '/etc/ssl/cert2.pem'] + ]) def test_add_http_directives(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) @@ -165,17 +169,19 @@ class NginxParserTest(util.NginxTest): target = set(['.example.com', 'example.*']) filep = nparser.abs_path('sites-enabled/example.com') nparser.add_server_directives( - filep, target, [['server_name', 'foo bar']], True) + filep, target, [['server_name', 'foobar.com']], replace=True) self.assertEqual( nparser.parsed[filep], [[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], - ['server_name', 'foo bar'], - ['server_name', 'foo bar']]]]) + ['server_name', 'foobar.com'], + ['server_name', 'example.*'], + ]]]) self.assertRaises(errors.MisconfigurationError, nparser.add_server_directives, - filep, set(['foo', 'bar']), - [['ssl_certificate', 'cert.pem']], True) + filep, set(['foobar.com', 'example.*']), + [['ssl_certificate', 'cert.pem']], + replace=True) def test_get_best_match(self): target_name = 'www.eff.org' @@ -217,7 +223,8 @@ class NginxParserTest(util.NginxTest): set(['.example.com', 'example.*']), [['ssl_certificate', 'foo.pem'], ['ssl_certificate_key', 'bar.key'], - ['listen', '443 ssl']]) + ['listen', '443 ssl']], + replace=False) c_k = nparser.get_all_certs_keys() self.assertEqual(set([('foo.pem', 'bar.key', filep)]), c_k) diff --git a/letsencrypt-nginx/tests/boulder-integration.conf.sh b/letsencrypt-nginx/tests/boulder-integration.conf.sh index 12610d895..d77669a76 100755 --- a/letsencrypt-nginx/tests/boulder-integration.conf.sh +++ b/letsencrypt-nginx/tests/boulder-integration.conf.sh @@ -20,13 +20,14 @@ events { } http { - # Set an array of temp and cache file options that will otherwise default to + # Set an array of temp, cache and log file options that will otherwise default to # restricted locations accessible only to root. client_body_temp_path $root/client_body; fastcgi_temp_path $root/fastcgi_temp; proxy_temp_path $root/proxy_temp; #scgi_temp_path $root/scgi_temp; #uwsgi_temp_path $root/uwsgi_temp; + access_log $root/error.log; # This should be turned off in a Virtualbox VM, as it can cause some # interesting issues with data corruption in delivered files. @@ -54,9 +55,6 @@ http { root $root/webroot; - access_log $root/access.log; - error_log $root/error.log; - location / { # First attempt to serve request as file, then as directory, then fall # back to index.html. From dc3a2da9b11b26be54b035ac1520128e8efbfe22 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 3 Jan 2016 10:49:50 -0500 Subject: [PATCH 37/62] Fixed a typo in a comment --- acme/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index ba2c88394..7314152cd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -31,7 +31,7 @@ else: install_requires.append('mock') if sys.version_info < (2, 7, 9): - # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) + # For secure SSL connection with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') install_requires.append('pyasn1') From 0454031cce4b88fef44e3e129e879a35b49c2314 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 3 Jan 2016 14:37:08 -0500 Subject: [PATCH 38/62] Fixed a pair of typos in docstrings --- acme/acme/jose/json_util.py | 2 +- acme/acme/jose/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index 7b95e3fce..977a06622 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -226,7 +226,7 @@ class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): :param str name: Name of the field to be encoded. - :raises erors.SerializationError: if field cannot be serialized + :raises errors.SerializationError: if field cannot be serialized :raises errors.Error: if field could not be found """ diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index ab3606efc..600077b20 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -130,7 +130,7 @@ class ImmutableMap(collections.Mapping, collections.Hashable): """Immutable key to value mapping with attribute access.""" __slots__ = () - """Must be overriden in subclasses.""" + """Must be overridden in subclasses.""" def __init__(self, **kwargs): if set(kwargs) != set(self.__slots__): From e722a381978afe3faaeb265eefbfc45b9a83e35f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 4 Jan 2016 11:18:13 -0800 Subject: [PATCH 39/62] Clarify parser check for duplicate values. --- letsencrypt-nginx/letsencrypt_nginx/parser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index 1d424f834..c60d0102a 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -518,8 +518,12 @@ def _add_directive(block, directive, replace): # Append directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. - if location != -1 and directive[0].__str__() not in repeatable_directives: - if block[location][1] == directive[1]: + directive_name = directive[0] + directive_value = directive[1] + if location != -1 and directive_name.__str__() not in repeatable_directives: + if block[location][1] == directive_value: + # There's a conflict, but the existing value matches the one we + # want to insert, so it's fine. pass else: raise errors.MisconfigurationError( From 4d3d6ff03157fe9ec77ba60e2fd59cca78ee8ce9 Mon Sep 17 00:00:00 2001 From: watercrossing Date: Tue, 5 Jan 2016 16:51:34 +0000 Subject: [PATCH 40/62] Fix for listen bug --- .../letsencrypt_apache/configurator.py | 2 ++ .../tests/configurator_test.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 836d77135..ff12055ac 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -558,6 +558,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # In case no Listens are set (which really is a broken apache config) if not listens: listens = ["80"] + if port in listens: + return for listen in listens: # For any listen statement, check if the machine also listens on Port 443. # If not, add such a listen statement. diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 218c085f9..9838b4f52 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -461,6 +461,25 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(mock_add_dir.call_args_list[1][0][2], ["[::1]:8080", "https"]) self.assertEqual(mock_add_dir.call_args_list[2][0][2], ["1.1.1.1:8080", "https"]) + def test_prepare_server_https_mixed_listen(self): + + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:8080", "443"] + mock_add_dir = mock.Mock() + mock_enable = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir_to_ifmodssl = mock_add_dir + self.config.enable_mod = mock_enable + + # Test Listen statements with specific ip listeed + self.config.prepare_server_https("443") + # Should only be 2 here, as the third interface already listens to the correct port + self.assertEqual(mock_add_dir.call_count, 0) + def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) From f7fad9a4affc399a4c6953e330d59b20d44d9d2e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 23 Dec 2015 23:25:41 -0800 Subject: [PATCH 41/62] Show error detail to the user. Previously this code would show the user a hardcoded message based on the error type, but it's much more useful to show the error detail field, which often helps debug the problem. --- letsencrypt/auth_handler.py | 7 +------ letsencrypt/tests/auth_handler_test.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 027c11158..37b71775d 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -543,13 +543,8 @@ def _generate_failed_chall_msg(failed_achalls): msg = [ "The following '{0}' errors were reported by the server:".format(typ)] - problems = dict() for achall in failed_achalls: - problems.setdefault(achall.error.description, set()).add(achall.domain) - for problem in problems: - msg.append("\n\nDomains: ") - msg.append(", ".join(sorted(problems[problem]))) - msg.append("\nError: {0}".format(problem)) + msg.append("\n\nDomain: %s\nDetail: %s\n" % (achall.domain, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/tests/auth_handler_test.py b/letsencrypt/tests/auth_handler_test.py index be19ab036..199954278 100644 --- a/letsencrypt/tests/auth_handler_test.py +++ b/letsencrypt/tests/auth_handler_test.py @@ -467,7 +467,7 @@ class ReportFailedChallsTest(unittest.TestCase): auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) - self.assertTrue("Domains: example.com\n" in call_list[0][0][0]) + self.assertTrue("Domain: example.com\nDetail: detail" in call_list[0][0][0]) @mock.patch("letsencrypt.auth_handler.zope.component.getUtility") def test_different_errors_and_domains(self, mock_zope): From 4adc2826318168f8318a7784a5b190599a03a045 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 5 Jan 2016 11:02:03 -0800 Subject: [PATCH 42/62] Fix formatting of error messages. --- letsencrypt/auth_handler.py | 4 ++-- letsencrypt/tests/auth_handler_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 37b71775d..e77f83248 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -541,10 +541,10 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ msg = [ - "The following '{0}' errors were reported by the server:".format(typ)] + "The following errors were reported by the server:".format(typ)] for achall in failed_achalls: - msg.append("\n\nDomain: %s\nDetail: %s\n" % (achall.domain, achall.error.detail)) + msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % (achall.domain, achall.error.typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/tests/auth_handler_test.py b/letsencrypt/tests/auth_handler_test.py index 199954278..5b4c2bfc7 100644 --- a/letsencrypt/tests/auth_handler_test.py +++ b/letsencrypt/tests/auth_handler_test.py @@ -467,7 +467,7 @@ class ReportFailedChallsTest(unittest.TestCase): auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) - self.assertTrue("Domain: example.com\nDetail: detail" in call_list[0][0][0]) + self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @mock.patch("letsencrypt.auth_handler.zope.component.getUtility") def test_different_errors_and_domains(self, mock_zope): From 74237d101041897231881a5ba55142e7fb255a4d Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 12:00:20 -0500 Subject: [PATCH 43/62] Requires chain_path for nginx versions supporting OCSP stapling --chain-path config is not mandatory, so we require this property if nginx supports OCSP stapling. Alternatively, we could disable OCSP stapling on supported nginx versions if --chain-path is missing. --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 11 ++++++++++- .../letsencrypt_nginx/tests/configurator_test.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index aaaf43c5f..99a864141 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -122,7 +122,7 @@ class NginxConfigurator(common.Plugin): # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, - chain_path, fullchain_path): + chain_path=None, fullchain_path=None): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. @@ -136,6 +136,9 @@ class NginxConfigurator(common.Plugin): .. note:: This doesn't save the config files! + :raises errors.PluginError: When unable to deploy certificate due to + a lack of directives or configuration + """ vhost = self.choose_vhost(domain) cert_directives = [['ssl_certificate', fullchain_path], @@ -150,6 +153,12 @@ class NginxConfigurator(common.Plugin): ['ssl_stapling', 'on'], ['ssl_stapling_verify', 'on']] + if len(stapling_directives) != 0 and not chain_path: + raise errors.PluginError( + "--chain-path is required to enable " + "Online Certificate Status Protocol (OCSP) stapling " + "on nginx >= 1.3.7.") + try: self.parser.add_server_directives(vhost.filep, vhost.names, cert_directives, replace=True) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..35a55befd 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -125,6 +125,15 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(util.contains_at_depth(generated_conf, ['ssl_trusted_certificate', 'example/chain.pem'], 2)) + def test_deploy_cert_stapling_requires_chain_path(self): + self.config.version = (1, 3, 7) + self.assertRaises(errors.PluginError, self.config.deploy_cert, + "www.example.com", + "example/cert.pem", + "example/key.pem", + None, + "example/fullchain.pem") + def test_deploy_cert(self): server_conf = self.config.parser.abs_path('server.conf') nginx_conf = self.config.parser.abs_path('nginx.conf') From e8fc2eca0139df1d8cde1e23c12eaca184371100 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 15:02:15 -0500 Subject: [PATCH 44/62] nginx plugin requires fullchain_path --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 4 ++++ .../letsencrypt_nginx/tests/configurator_test.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 99a864141..59a977f09 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -140,6 +140,10 @@ class NginxConfigurator(common.Plugin): a lack of directives or configuration """ + if not fullchain_path: + raise errors.PluginError( + "--fullchain-path is required for nginx plugin.") + vhost = self.choose_vhost(domain) cert_directives = [['ssl_certificate', fullchain_path], ['ssl_certificate_key', key_path]] diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 35a55befd..bab107582 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -134,6 +134,15 @@ class NginxConfiguratorTest(util.NginxTest): None, "example/fullchain.pem") + def test_deploy_cert_requires_fullchain_path(self): + self.config.version = (1, 3, 1) + self.assertRaises(errors.PluginError, self.config.deploy_cert, + "www.example.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + None) + def test_deploy_cert(self): server_conf = self.config.parser.abs_path('server.conf') nginx_conf = self.config.parser.abs_path('nginx.conf') From 5b5051b6ced180c10311cea89b37900d147929c1 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 15:16:30 -0500 Subject: [PATCH 45/62] The notes should display the fullchain_path See d01b17f1 and dd8c6d65 --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 59a977f09..963ae8a42 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -181,7 +181,7 @@ class NginxConfigurator(common.Plugin): self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, ", ".join(str(addr) for addr in vhost.addrs))) - self.save_notes += "\tssl_certificate %s\n" % cert_path + self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path ####################### From 0b9f505ed5beb7a861c9cb027b5e4349a67eb60d Mon Sep 17 00:00:00 2001 From: Fan Jiang Date: Mon, 4 Jan 2016 15:29:26 -0500 Subject: [PATCH 46/62] update document for --chain-path when required by Nginx >= 1.3.7 --- docs/using.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 5da13f02c..bd40dec02 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -237,7 +237,9 @@ The following files are available: server certificate, i.e. root and intermediate certificates only. This is what Apache < 2.4.8 needs for `SSLCertificateChainFile - `_. + `_, + and what nginx >= 1.3.7 needs for `ssl_trusted_certificate + `_. ``fullchain.pem`` All certificates, **including** server certificate. This is From cf74446b58fa1229c6b2ad6bf5465f3b9ebe41ef Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 23:04:18 -0500 Subject: [PATCH 47/62] Add test for redirect enhancement --- .../letsencrypt_nginx/tests/configurator_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..5e237b04a 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -330,6 +330,17 @@ class NginxConfiguratorTest(util.NginxTest): OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, key_file.read()) + def test_redirect_enhance(self): + expected = [ + ['if', '($scheme != "https")'], + [['return', '301 https://$host$request_uri']] + ] + + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.enhance("www.example.com", "redirect") + + generated_conf = self.config.parser.parsed[example_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) if __name__ == "__main__": unittest.main() # pragma: no cover From adcb7934aed5780ba96e9f5c861c467bff8f616d Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 22:48:01 -0500 Subject: [PATCH 48/62] Improve choose_vhost() test by verifying the output file --- .../tests/configurator_test.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 5e237b04a..9a962a55f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -91,12 +91,26 @@ class NginxConfiguratorTest(util.NginxTest): 'test.www.example.com': foo_conf, 'abc.www.foo.com': foo_conf, 'www.bar.co.uk': localhost_conf} + + conf_path = {'localhost': "etc_nginx/nginx.conf", + 'alias': "etc_nginx/nginx.conf", + 'example.com': "etc_nginx/sites-enabled/example.com", + 'example.com.uk.test': "etc_nginx/sites-enabled/example.com", + 'www.example.com': "etc_nginx/sites-enabled/example.com", + 'test.www.example.com': "etc_nginx/foo.conf", + 'abc.www.foo.com': "etc_nginx/foo.conf", + 'www.bar.co.uk': "etc_nginx/nginx.conf"} + bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] for name in results: - self.assertEqual(results[name], - self.config.choose_vhost(name).names) + vhost = self.config.choose_vhost(name) + path = os.path.relpath(vhost.filep, self.temp_dir) + + self.assertEqual(results[name], vhost.names) + self.assertEqual(conf_path[name], path) + for name in bad_results: self.assertEqual(set([name]), self.config.choose_vhost(name).names) From 20829e05ed6a665a224fdef43e4f91cd5c002199 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Tue, 5 Jan 2016 15:15:29 -0500 Subject: [PATCH 49/62] Add missing test condition for prepare() --- .../tests/configurator_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 9a962a55f..3ad0b834f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -40,6 +40,23 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEquals((1, 6, 2), self.config.version) self.assertEquals(5, len(self.config.parser.parsed)) + @mock.patch("letsencrypt_nginx.configurator.le_util.exe_exists") + @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") + def test_prepare_initializes_version(self, mock_popen, mock_exe_exists): + mock_popen().communicate.return_value = ( + "", "\n".join(["nginx version: nginx/1.6.2", + "built by clang 6.0 (clang-600.0.56)" + " (based on LLVM 3.5svn)", + "TLS SNI support enabled", + "configure arguments: --prefix=/usr/local/Cellar/" + "nginx/1.6.2 --with-http_ssl_module"])) + + mock_exe_exists.return_value = True + + self.config.version = None + self.config.prepare() + self.assertEquals((1, 6, 2), self.config.version) + @mock.patch("letsencrypt_nginx.configurator.socket.gethostbyaddr") def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) From 0a04efe40c55029ba66a5c8b69f0d3f646286aab Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 5 Jan 2016 17:23:32 -0800 Subject: [PATCH 50/62] Fix lint error and remove extra format() --- letsencrypt/auth_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index e77f83248..c63d8c8d4 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -540,11 +540,11 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ - msg = [ - "The following errors were reported by the server:".format(typ)] + msg = ["The following errors were reported by the server:"] for achall in failed_achalls: - msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % (achall.domain, achall.error.typ, achall.error.detail)) + msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % ( + achall.domain, achall.error.typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") From 858dadd85b2b387f8e5c0485e370efa57bb7fe18 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Wed, 6 Jan 2016 13:36:52 -0500 Subject: [PATCH 51/62] Update error message This is supposed to not happen once #1391 is fixed. --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 963ae8a42..43ad36c7f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -142,7 +142,8 @@ class NginxConfigurator(common.Plugin): """ if not fullchain_path: raise errors.PluginError( - "--fullchain-path is required for nginx plugin.") + "The nginx plugin currently requires --fullchain-path to " + "install a cert.") vhost = self.choose_vhost(domain) cert_directives = [['ssl_certificate', fullchain_path], From 9dc4af5cee5445e1a8bdfab97d56ecbceea6d659 Mon Sep 17 00:00:00 2001 From: Ben Ubois Date: Wed, 6 Jan 2016 12:10:21 -0800 Subject: [PATCH 52/62] Document webroot request path. It's handy to know the implementation details of the webroot plugin so that a server can be configured to properly the ACME challenge files. --- docs/using.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/using.rst b/docs/using.rst index 5da13f02c..f7a59cf7c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -139,9 +139,20 @@ Would obtain a single certificate for all of those names, using the ``/var/www/example`` webroot directory for the first two, and ``/var/www/eg`` for the second two. +The webroot plugin works by creating a temporary file for each of your requested +domains in ``${webroot-path}/.well-known/acme-challenge``. Then the Let's +Encrypt validation server makes HTTP requests to validate that the DNS for each +requested domain resolves to the server running letsencrypt. An example request +made to your web server would look like: + +:: + + 66.133.109.36 - - [05/Jan/2016:20:11:24 -0500] "GET /.well-known/acme-challenge/HGr8U1IeTW4kY_Z6UIyaakzOkyQgPr_7ArlLgtZE8SX HTTP/1.1" 200 87 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" + Note that to use the webroot plugin, your server must be configured to serve files from hidden directories. + Manual ------ From e7da21eec0cadfe80f8836d44f1aa64cd89130de Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Thu, 7 Jan 2016 11:27:40 -0500 Subject: [PATCH 53/62] Makes NginxParser aware of directive Fixes #2059 --- letsencrypt-nginx/letsencrypt_nginx/parser.py | 6 ++++-- .../letsencrypt_nginx/tests/parser_test.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index c60d0102a..3b1dd049e 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -113,7 +113,7 @@ class NginxParser(object): for filename in servers: for server in servers[filename]: # Parse the server block into a VirtualHost object - parsed_server = _parse_server(server) + parsed_server = parse_server(server) vhost = obj.VirtualHost(filename, parsed_server['addrs'], parsed_server['ssl'], @@ -451,7 +451,7 @@ def _get_servernames(names): return names.split(' ') -def _parse_server(server): +def parse_server(server): """Parses a list of server directives. :param list server: list of directives in a server block @@ -471,6 +471,8 @@ def _parse_server(server): elif directive[0] == 'server_name': parsed_server['names'].update( _get_servernames(directive[1])) + elif directive[0] == 'ssl' and directive[1] == 'on': + parsed_server['ssl'] = True return parsed_server diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 6559a5df6..b64f1dee3 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -228,6 +228,26 @@ class NginxParserTest(util.NginxTest): c_k = nparser.get_all_certs_keys() self.assertEqual(set([('foo.pem', 'bar.key', filep)]), c_k) + def test_parse_server_ssl(self): + server = parser.parse_server([ + ['listen', '443'] + ]) + self.assertFalse(server['ssl']) + + server = parser.parse_server([ + ['listen', '443 ssl'] + ]) + self.assertTrue(server['ssl']) + + server = parser.parse_server([ + ['listen', '443'], ['ssl', 'off'] + ]) + self.assertFalse(server['ssl']) + + server = parser.parse_server([ + ['listen', '443'], ['ssl', 'on'] + ]) + self.assertTrue(server['ssl']) if __name__ == "__main__": unittest.main() # pragma: no cover From 6548f343bfe8b9fc3e00e1ef2abe356ff4c3c13c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 21:20:14 +0000 Subject: [PATCH 54/62] Add invalidEmail error type to acme Related to: - #1923 - https://github.com/ietf-wg-acme/acme/pull/65 --- acme/acme/messages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 0b73864ec..3b5739da1 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -25,6 +25,8 @@ class Error(jose.JSONObjectWithFields, errors.Error): ('connection', 'The server could not connect to the client to ' 'verify the domain'), ('dnssec', 'The server could not validate a DNSSEC signed domain'), + ('invalidEmail', + 'The provided email for a registration was invalid'), ('malformed', 'The request message was malformed'), ('rateLimited', 'There were too many requests of a given type'), ('serverInternal', 'The server experienced an internal error'), From 6fedd22dc8c2db00665fa4343779f4aa801d571e Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Jan 2016 02:00:47 -0800 Subject: [PATCH 55/62] don't iDisplay if logging --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt/reverter.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 1baa06128..3a0b90fc0 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1271,7 +1271,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self.config_test() - logger.debug(self.reverter.view_config_changes()) + logger.debug(self.reverter.view_config_changes(logging=True)) self._reload() def _reload(self): diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index d5114ae71..e2bc28cb9 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -94,7 +94,7 @@ class Reverter(object): "Unable to load checkpoint during rollback") rollback -= 1 - def view_config_changes(self): + def view_config_changes(self, logging=False): """Displays all saved checkpoints. All checkpoints are printed by @@ -144,6 +144,8 @@ class Reverter(object): output.append(os.linesep) + if logging: + return os.linesep.join(output) zope.component.getUtility(interfaces.IDisplay).notification( os.linesep.join(output), display_util.HEIGHT) From 287be6be8eeac0e4da55c6088bda74ec3d48a8d7 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Jan 2016 02:47:59 -0800 Subject: [PATCH 56/62] fix linting issue --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt/reverter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index d936309ae..2a9fb0250 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1302,7 +1302,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self.config_test() - logger.debug(self.reverter.view_config_changes(logging=True)) + logger.debug(self.reverter.view_config_changes(for_logging=True)) self._reload() def _reload(self): diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index e2bc28cb9..863074374 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -94,7 +94,7 @@ class Reverter(object): "Unable to load checkpoint during rollback") rollback -= 1 - def view_config_changes(self, logging=False): + def view_config_changes(self, for_logging=False): """Displays all saved checkpoints. All checkpoints are printed by @@ -144,7 +144,7 @@ class Reverter(object): output.append(os.linesep) - if logging: + if for_logging: return os.linesep.join(output) zope.component.getUtility(interfaces.IDisplay).notification( os.linesep.join(output), display_util.HEIGHT) From 2c8d042974dbbf78a03a95dad187ed39c867664a Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Jan 2016 04:58:17 -0800 Subject: [PATCH 57/62] added in tls_sni logging from #2002 --- letsencrypt-apache/letsencrypt_apache/tls_sni_01.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index 2049eb574..ca7985f35 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -1,12 +1,14 @@ """A class that performs TLS-SNI-01 challenges for Apache""" import os +import logging from letsencrypt.plugins import common from letsencrypt_apache import obj from letsencrypt_apache import parser +logger = logging.getLogger(__name__) class ApacheTlsSni01(common.TLSSNI01): """Class that performs TLS-SNI-01 challenges within the Apache configurator @@ -104,6 +106,7 @@ class ApacheTlsSni01(common.TLSSNI01): self.configurator.reverter.register_file_creation( True, self.challenge_conf) + logger.debug("writing a config file with text: %s", config_text) with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) From b039c884d8ffa16e2b3ac9663cca634bfd2f8bc3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 8 Jan 2016 14:09:44 -0500 Subject: [PATCH 58/62] Don't use cryptography version 1.2 --- acme/setup.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7314152cd..54c4d82d9 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,7 +9,7 @@ version = '0.2.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8', + 'cryptography>=0.8,<1.2', # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15) 'PyOpenSSL>=0.15', 'pyrfc3339', diff --git a/setup.py b/setup.py index f95f672ff..4005b0973 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ version = meta['version'] install_requires = [ 'acme=={0}'.format(version), 'configobj', - 'cryptography>=0.7', # load_pem_x509_certificate + 'cryptography>=0.7,<1.2', # load_pem_x509_certificate 'parsedatetime', 'psutil>=2.1.0', # net_connections introduced in 2.1.0 'PyOpenSSL', From 96b55c8f349ac29e314e5468c406d0e07f4b4917 Mon Sep 17 00:00:00 2001 From: bmw Date: Fri, 8 Jan 2016 17:02:35 -0500 Subject: [PATCH 59/62] Revert "Don't use cryptography version 1.2" --- acme/setup.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 54c4d82d9..7314152cd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,7 +9,7 @@ version = '0.2.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8,<1.2', + 'cryptography>=0.8', # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15) 'PyOpenSSL>=0.15', 'pyrfc3339', diff --git a/setup.py b/setup.py index 4005b0973..f95f672ff 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ version = meta['version'] install_requires = [ 'acme=={0}'.format(version), 'configobj', - 'cryptography>=0.7,<1.2', # load_pem_x509_certificate + 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', 'psutil>=2.1.0', # net_connections introduced in 2.1.0 'PyOpenSSL', From 7728f4d28a14724783bd8ee42d54ab047d645500 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 9 Jan 2016 13:24:39 +0000 Subject: [PATCH 60/62] Python 3 Travis testing for acme. Despite its description, https://github.com/letsencrypt/letsencrypt/pull/630, removed not only Python 2.6 support, but also Travis tests against Python 3. ACME library supports Python 3 and Travis should tests it. This must be merged before any pending PRs agains acme library. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a5d6d8a85..6cadbd36e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ env: matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 + - TOXENV=py33 + - TOXENV=py34 + - TOXENV=py35 - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration From 34010a0168d0df97a08bdd5b985e7694a37123f2 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 9 Jan 2016 13:31:50 +0000 Subject: [PATCH 61/62] Python 3.5 needs explicit Travis setting --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6cadbd36e..680dfeb8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,12 +24,15 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=py33 - TOXENV=py34 - - TOXENV=py35 - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration # DNS timeouts :( # - TOXENV=apacheconftest +matrix: + include: + - env: TOXENV=py35 + python: 3.5 # Only build pushes to the master branch, PRs, and branches beginning with From b26dda3afe859ca6586d6d89d5b500ee416d2841 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 9 Jan 2016 13:38:12 +0000 Subject: [PATCH 62/62] Add Python 3.5 trove classifier to acme --- acme/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/setup.py b/acme/setup.py index 7314152cd..f585b3cdd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -66,6 +66,7 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ],