From c0048d7480e7ba8b62baed3bfded465146c85156 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Mon, 2 Nov 2015 16:22:58 -0800 Subject: [PATCH 1/5] load augeas httpd lens from inside of lets encrypt --- letsencrypt-apache/MANIFEST.in | 1 + .../letsencrypt_apache/augeas_configurator.py | 6 +- .../letsencrypt_apache/constants.py | 4 + .../letsencrypt_apache/httpd.aug | 112 ++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/httpd.aug diff --git a/letsencrypt-apache/MANIFEST.in b/letsencrypt-apache/MANIFEST.in index ff99bf0d8..0cedae323 100644 --- a/letsencrypt-apache/MANIFEST.in +++ b/letsencrypt-apache/MANIFEST.in @@ -2,3 +2,4 @@ include LICENSE.txt include README.rst recursive-include letsencrypt_apache/tests/testdata * include letsencrypt_apache/options-ssl-apache.conf +include letsencrypt_apache/httpd.aug diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 7557a27c6..53d1bc017 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -1,5 +1,6 @@ """Class of Augeas Configurators.""" import logging +import os import augeas @@ -7,6 +8,8 @@ from letsencrypt import errors from letsencrypt import reverter from letsencrypt.plugins import common +from letsencrypt_apache import constants + logger = logging.getLogger(__name__) @@ -30,7 +33,8 @@ class AugeasConfigurator(common.Plugin): # Set Augeas flags to not save backup (we do it ourselves) # Set Augeas to not load anything by default my_flags = augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD - self.aug = augeas.Augeas(flags=my_flags) + my_dir = os.path.dirname(constants.AUGEAS_HTTPD_LENS) + self.aug = augeas.Augeas(loadpath=my_dir, flags=my_flags) self.save_notes = "" # See if any temporary changes need to be recovered diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index b38e898cf..0a1053832 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -20,6 +20,10 @@ MOD_SSL_CONF_SRC = pkg_resources.resource_filename( """Path to the Apache mod_ssl config file found in the Let's Encrypt distribution.""" +AUGEAS_HTTPD_LENS = pkg_resources.resource_filename( + "letsencrypt_apache", "httpd.aug") +"""Path to the Augeas lens""" + REWRITE_HTTPS_ARGS = [ "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] """Apache rewrite rule arguments used for redirections to https vhost""" diff --git a/letsencrypt-apache/letsencrypt_apache/httpd.aug b/letsencrypt-apache/letsencrypt_apache/httpd.aug new file mode 100644 index 000000000..9b50a8f0e --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/httpd.aug @@ -0,0 +1,112 @@ +(* Apache HTTPD lens for Augeas + +Authors: + David Lutterkort + Francis Giraldeau + Raphael Pinson + +About: Reference + Online Apache configuration manual: http://httpd.apache.org/docs/trunk/ + +About: License + This file is licensed under the LGPL v2+. + +About: Lens Usage + Sample usage of this lens in augtool + + Apache configuration is represented by two main structures, nested sections + and directives. Sections are used as labels, while directives are kept as a + value. Sections and directives can have positional arguments inside values + of "arg" nodes. Arguments of sections must be the firsts child of the + section node. + + This lens doesn't support automatic string quoting. Hence, the string must + be quoted when containing a space. + + Create a new VirtualHost section with one directive: + > clear /files/etc/apache2/sites-available/foo/VirtualHost + > set /files/etc/apache2/sites-available/foo/VirtualHost/arg "172.16.0.1:80" + > set /files/etc/apache2/sites-available/foo/VirtualHost/directive "ServerAdmin" + > set /files/etc/apache2/sites-available/foo/VirtualHost/*[self::directive="ServerAdmin"]/arg "admin@example.com" + +About: Configuration files + This lens applies to files in /etc/httpd and /etc/apache2. See . + +*) + + +module Httpd = + +autoload xfm + +(****************************************************************** + * Utilities lens + *****************************************************************) +let dels (s:string) = del s s + +(* deal with continuation lines *) +let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " + +let sep_osp = Sep.opt_space +let sep_eq = del /[ \t]*=[ \t]*/ "=" + +let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ +let word = /[a-zA-Z][a-zA-Z0-9._-]*/ + +let comment = Util.comment +let eol = Util.doseol +let empty = Util.empty_dos +let indent = Util.indent + +(* borrowed from shellvars.aug *) +let char_arg_dir = /[^\\ '"\t\r\n]|\\\\"|\\\\'/ +let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ +let cdot = /\\\\./ +let cl = /\\\\\n/ +let dquot = + let no_dquot = /[^"\\\r\n]/ + in /"/ . (no_dquot|cdot|cl)* . /"/ +let squot = + let no_squot = /[^'\\\r\n]/ + in /'/ . (no_squot|cdot|cl)* . /'/ +let comp = /[<>=]?=/ + +(****************************************************************** + * Attributes + *****************************************************************) + +let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ] +let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] + +let argv (l:lens) = l . (sep_spc . l)* + +let directive = [ indent . label "directive" . store word . + (sep_spc . argv arg_dir)? . eol ] + +let section (body:lens) = + (* opt_eol includes empty lines *) + let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in + let inner = (sep_spc . argv arg_sec)? . sep_osp . + dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? . + indent . dels "" ">" . eol ] + +let rec content = section (content|directive) + +let lns = (content|directive|comment|empty)* + +let filter = (incl "/etc/apache2/apache2.conf") . + (incl "/etc/apache2/httpd.conf") . + (incl "/etc/apache2/ports.conf") . + (incl "/etc/apache2/conf.d/*") . + (incl "/etc/apache2/conf-available/*.conf") . + (incl "/etc/apache2/mods-available/*") . + (incl "/etc/apache2/sites-available/*") . + (incl "/etc/httpd/conf.d/*.conf") . + (incl "/etc/httpd/httpd.conf") . + (incl "/etc/httpd/conf/httpd.conf") . + Util.stdexcl + +let xfm = transform lns filter From 60fce0b22e422bbfd1bb700740c9648d209c0228 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 3 Nov 2015 11:08:10 -0800 Subject: [PATCH 2/5] switched the httpd.aug to the version from 1.2 --- .../letsencrypt_apache/httpd.aug | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/httpd.aug b/letsencrypt-apache/letsencrypt_apache/httpd.aug index 9b50a8f0e..caea9b67c 100644 --- a/letsencrypt-apache/letsencrypt_apache/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/httpd.aug @@ -59,24 +59,17 @@ let empty = Util.empty_dos let indent = Util.indent (* borrowed from shellvars.aug *) -let char_arg_dir = /[^\\ '"\t\r\n]|\\\\"|\\\\'/ +let char_arg_dir = /[^ '"\t\r\n]|\\\\"|\\\\'/ let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ -let cdot = /\\\\./ -let cl = /\\\\\n/ -let dquot = - let no_dquot = /[^"\\\r\n]/ - in /"/ . (no_dquot|cdot|cl)* . /"/ -let squot = - let no_squot = /[^'\\\r\n]/ - in /'/ . (no_squot|cdot|cl)* . /'/ -let comp = /[<>=]?=/ +let dquot = /"([^"\\\r\n]|\\\\.)*"/ +let squot = /'([^'\\\r\n]|\\\\.)*'/ (****************************************************************** * Attributes *****************************************************************) let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ] -let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] +let arg_sec = [ label "arg" . store (char_arg_sec+|dquot|squot) ] let argv (l:lens) = l . (sep_spc . l)* @@ -84,16 +77,13 @@ let directive = [ indent . label "directive" . store word . (sep_spc . argv arg_dir)? . eol ] let section (body:lens) = - (* opt_eol includes empty lines *) - let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in let inner = (sep_spc . argv arg_sec)? . sep_osp . - dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? . - indent . dels "" . eol . body* . indent . dels "" ">" . eol ] -let rec content = section (content|directive) +let rec content = section (content|directive|comment|empty) let lns = (content|directive|comment|empty)* @@ -101,7 +91,6 @@ let filter = (incl "/etc/apache2/apache2.conf") . (incl "/etc/apache2/httpd.conf") . (incl "/etc/apache2/ports.conf") . (incl "/etc/apache2/conf.d/*") . - (incl "/etc/apache2/conf-available/*.conf") . (incl "/etc/apache2/mods-available/*") . (incl "/etc/apache2/sites-available/*") . (incl "/etc/httpd/conf.d/*.conf") . From f5053ba8af122ac91b60e833b0c1c6576e226825 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 4 Nov 2015 20:12:39 +0000 Subject: [PATCH 3/5] constants.AUGEAS_LENS_DIR --- letsencrypt-apache/MANIFEST.in | 2 +- .../letsencrypt_apache/augeas_configurator.py | 10 +++++----- .../letsencrypt_apache/{ => augeas_lens}/httpd.aug | 0 letsencrypt-apache/letsencrypt_apache/constants.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) rename letsencrypt-apache/letsencrypt_apache/{ => augeas_lens}/httpd.aug (100%) diff --git a/letsencrypt-apache/MANIFEST.in b/letsencrypt-apache/MANIFEST.in index 15d5af8cd..933cc10ac 100644 --- a/letsencrypt-apache/MANIFEST.in +++ b/letsencrypt-apache/MANIFEST.in @@ -3,4 +3,4 @@ include README.rst recursive-include docs * recursive-include letsencrypt_apache/tests/testdata * include letsencrypt_apache/options-ssl-apache.conf -include letsencrypt_apache/httpd.aug +recursive-include letsencrypt_apache/augeas_lens *.aug diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 53d1bc017..9b7e288d4 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -30,11 +30,11 @@ class AugeasConfigurator(common.Plugin): def __init__(self, *args, **kwargs): super(AugeasConfigurator, self).__init__(*args, **kwargs) - # Set Augeas flags to not save backup (we do it ourselves) - # Set Augeas to not load anything by default - my_flags = augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD - my_dir = os.path.dirname(constants.AUGEAS_HTTPD_LENS) - self.aug = augeas.Augeas(loadpath=my_dir, flags=my_flags) + self.aug = augeas.Augeas( + loadpath=constants.AUGEAS_LENS_DIR, + # Do not save backup (we do it ourselves), do not load + # anything by default + flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)) self.save_notes = "" # See if any temporary changes need to be recovered diff --git a/letsencrypt-apache/letsencrypt_apache/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug similarity index 100% rename from letsencrypt-apache/letsencrypt_apache/httpd.aug rename to letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index 0a1053832..1c17eacc3 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -20,9 +20,9 @@ MOD_SSL_CONF_SRC = pkg_resources.resource_filename( """Path to the Apache mod_ssl config file found in the Let's Encrypt distribution.""" -AUGEAS_HTTPD_LENS = pkg_resources.resource_filename( - "letsencrypt_apache", "httpd.aug") -"""Path to the Augeas lens""" +AUGEAS_LENS_DIR = pkg_resources.resource_filename( + "letsencrypt_apache", "augeas_lens") +"""Path to the Augeas lens directory""" REWRITE_HTTPS_ARGS = [ "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] From af5ab05a58b69b935c68d489dfd9cfabeab6ce39 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 4 Nov 2015 19:34:24 -0800 Subject: [PATCH 4/5] Revert "switched the httpd.aug to the version from 1.2" This reverts commit 60fce0b22e422bbfd1bb700740c9648d209c0228. --- .../letsencrypt_apache/augeas_lens/httpd.aug | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index caea9b67c..9b50a8f0e 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -59,17 +59,24 @@ let empty = Util.empty_dos let indent = Util.indent (* borrowed from shellvars.aug *) -let char_arg_dir = /[^ '"\t\r\n]|\\\\"|\\\\'/ +let char_arg_dir = /[^\\ '"\t\r\n]|\\\\"|\\\\'/ let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/ -let dquot = /"([^"\\\r\n]|\\\\.)*"/ -let squot = /'([^'\\\r\n]|\\\\.)*'/ +let cdot = /\\\\./ +let cl = /\\\\\n/ +let dquot = + let no_dquot = /[^"\\\r\n]/ + in /"/ . (no_dquot|cdot|cl)* . /"/ +let squot = + let no_squot = /[^'\\\r\n]/ + in /'/ . (no_squot|cdot|cl)* . /'/ +let comp = /[<>=]?=/ (****************************************************************** * Attributes *****************************************************************) let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ] -let arg_sec = [ label "arg" . store (char_arg_sec+|dquot|squot) ] +let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] let argv (l:lens) = l . (sep_spc . l)* @@ -77,13 +84,16 @@ let directive = [ indent . label "directive" . store word . (sep_spc . argv arg_dir)? . eol ] let section (body:lens) = + (* opt_eol includes empty lines *) + let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in let inner = (sep_spc . argv arg_sec)? . sep_osp . - dels ">" . eol . body* . indent . dels "" . opt_eol . ((body|comment) . (body|empty|comment)*)? . + indent . dels "" ">" . eol ] -let rec content = section (content|directive|comment|empty) +let rec content = section (content|directive) let lns = (content|directive|comment|empty)* @@ -91,6 +101,7 @@ let filter = (incl "/etc/apache2/apache2.conf") . (incl "/etc/apache2/httpd.conf") . (incl "/etc/apache2/ports.conf") . (incl "/etc/apache2/conf.d/*") . + (incl "/etc/apache2/conf-available/*.conf") . (incl "/etc/apache2/mods-available/*") . (incl "/etc/apache2/sites-available/*") . (incl "/etc/httpd/conf.d/*.conf") . From 5fd4ed9d9d6346cb6235f83bc6d1e0733b851e80 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 4 Nov 2015 19:43:11 -0800 Subject: [PATCH 5/5] added documentation explaining why we're loading lens --- letsencrypt-apache/letsencrypt_apache/augeas_configurator.py | 2 +- letsencrypt-apache/letsencrypt_apache/augeas_lens/README | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/augeas_lens/README diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 9b7e288d4..b0b15d649 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -1,6 +1,5 @@ """Class of Augeas Configurators.""" import logging -import os import augeas @@ -31,6 +30,7 @@ class AugeasConfigurator(common.Plugin): super(AugeasConfigurator, self).__init__(*args, **kwargs) self.aug = augeas.Augeas( + # specify a directory to load our preferred lens from loadpath=constants.AUGEAS_LENS_DIR, # Do not save backup (we do it ourselves), do not load # anything by default diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/README b/letsencrypt-apache/letsencrypt_apache/augeas_lens/README new file mode 100644 index 000000000..fc803a776 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/README @@ -0,0 +1,2 @@ +Let's Encrypt includes the very latest Augeas lenses in order to ship bug fixes +to Apacche configuration handling bugs as quickly as possible