Merge pull request #683 from letsencrypt/letshelp-apache

Letshelp apache
This commit is contained in:
James Kasten 2015-08-14 00:34:32 -04:00
commit 9abb56cb6e
12 changed files with 580 additions and 4 deletions

View file

@ -0,0 +1 @@
recursive-include letshelp-letsencrypt/testdata *

View file

@ -0,0 +1 @@
"""Tools for submitting server configurations"""

View file

@ -0,0 +1,303 @@
#!/usr/bin/env python
"""Let's Encrypt Apache configuration submission script"""
import argparse
import atexit
import contextlib
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import textwrap
_DESCRIPTION = """
Let's Help is a simple script you can run to help out the Let's Encrypt
project. Since Let's Encrypt will support automatically configuring HTTPS on
many servers, we want to test this functionality on as many configurations as
possible. This script will create a sanitized copy of your Apache
configuration, notifying you of the files that have been selected. If (and only
if) you approve this selection, these files will be sent to the Let's Encrypt
developers.
"""
_NO_APACHECTL = """
Unable to find `apachectl` which is required for this script to work. If it is
installed, please run this script again with the --apache-ctl command line
argument and the path to the binary.
"""
# Keywords likely to be found in filenames of sensitive files
_SENSITIVE_FILENAME_REGEX = re.compile(r"^(?!.*proxy_fdpass).*pass.*$|private|"
r"secret|cert|crt|key|rsa|dsa|pw|\.pem|"
r"\.der|\.p12|\.pfx|\.p7b")
def make_and_verify_selection(server_root, temp_dir):
"""Copies server_root to temp_dir and verifies selection with the user
:param str server_root: Path to the Apache server root
:param str temp_dir: Path to the temporary directory to copy files to
"""
copied_files, copied_dirs = copy_config(server_root, temp_dir)
print textwrap.fill("A secure copy of the files that have been selected "
"for submission has been created under {0}. All "
"comments have been removed and the files are only "
"accessible by the current user. A list of the files "
"that have been included is shown below. Please make "
"sure that this selection does not contain private "
"keys, passwords, or any other sensitive "
"information.".format(temp_dir))
print "\nFiles:"
for copied_file in copied_files:
print copied_file
print "Directories (including all contained files):"
for copied_dir in copied_dirs:
print copied_dir
sys.stdout.write("\nIs it safe to submit these files? ")
while True:
ans = raw_input("(Y)es/(N)o: ").lower()
if ans.startswith("y"):
return
elif ans.startswith("n"):
sys.exit("Your files were not submitted")
def copy_config(server_root, temp_dir):
"""Safely copies server_root to temp_dir and returns copied files
:param str server_root: Absolute path to the Apache server root
:param str temp_dir: Path to the temporary directory to copy files to
:returns: List of copied files and a list of leaf directories where
all contained files were copied
:rtype: `tuple` of `list` of `str`
"""
copied_files, copied_dirs = [], []
dir_len = len(os.path.dirname(server_root))
for config_path, config_dirs, config_files in os.walk(server_root):
temp_path = os.path.join(temp_dir, config_path[dir_len+1:])
os.mkdir(temp_path)
copied_all = True
copied_files_in_current_dir = []
for config_file in config_files:
config_file_path = os.path.join(config_path, config_file)
temp_file_path = os.path.join(temp_path, config_file)
if os.path.islink(config_file_path):
os.symlink(os.readlink(config_file_path), temp_file_path)
elif safe_config_file(config_file_path):
copy_file_without_comments(config_file_path, temp_file_path)
copied_files_in_current_dir.append(config_file_path)
else:
copied_all = False
# If copied all files in leaf directory
if copied_all and not config_dirs:
copied_dirs.append(config_path)
else:
copied_files += copied_files_in_current_dir
return copied_files, copied_dirs
def copy_file_without_comments(source, destination):
"""Copies source to destination, removing comments
:param str source: Path to the file to be copied
:param str destination: Path where source should be copied to
"""
with open(source, "r") as infile:
with open(destination, "w") as outfile:
for line in infile:
if not (line.isspace() or line.lstrip().startswith("#")):
outfile.write(line)
def safe_config_file(config_file):
"""Returns True if config_file can be safely copied
:param str config_file: Path to an Apache configuration file
:returns: True if config_file can be safely copied
:rtype: bool
"""
config_file_lower = config_file.lower()
if _SENSITIVE_FILENAME_REGEX.search(config_file_lower):
return False
proc = subprocess.Popen(["file", config_file],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
file_output, _ = proc.communicate()
if "ASCII" in file_output:
possible_password_file = empty_or_all_comments = True
with open(config_file) as config_fd:
for line in config_fd:
if not (line.isspace() or line.lstrip().startswith("#")):
empty_or_all_comments = False
if line.startswith("-----BEGIN"):
return False
elif not ":" in line:
possible_password_file = False
# If file isn't empty or commented out and could be a password file,
# don't include it in selection. It is safe to include the file if
# it consists solely of comments because comments are removed before
# submission.
return empty_or_all_comments or not possible_password_file
return False
def setup_tempdir(args):
"""Creates a temporary directory and necessary files for config
:param argparse.Namespace args: Parsed command line arguments
:returns: Path to temporary directory
:rtype: str
"""
tempdir = tempfile.mkdtemp()
with open(os.path.join(tempdir, "config_file"), "w") as config_fd:
config_fd.write(args.config_file + "\n")
proc = subprocess.Popen([args.apache_ctl, "-v"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
with open(os.path.join(tempdir, "version"), "w") as version_fd:
version_fd.write(proc.communicate()[0])
proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f",
args.config_file, "-M"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
with open(os.path.join(tempdir, "modules"), "w") as modules_fd:
modules_fd.write(proc.communicate()[0])
proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f",
args.config_file, "-t", "-D", "DUMP_VHOSTS"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
with open(os.path.join(tempdir, "vhosts"), "w") as vhosts_fd:
vhosts_fd.write(proc.communicate()[0])
return tempdir
def verify_config(args):
"""Verifies server_root and config_file specify a valid config
:param argparse.Namespace args: Parsed command line arguments
"""
with open(os.devnull, "w") as devnull:
try:
subprocess.check_call([args.apache_ctl, "-d", args.server_root,
"-f", args.config_file, "-t"],
stdout=devnull, stderr=subprocess.STDOUT)
except OSError:
sys.exit(_NO_APACHECTL)
except subprocess.CalledProcessError:
sys.exit("Syntax check from apachectl failed")
def locate_config(apache_ctl):
"""Uses the apachectl binary to find configuration files
:param str apache_ctl: Path to `apachectl` binary
:returns: Path to Apache server root and main configuration file
:rtype: `tuple` of `str`
"""
try:
proc = subprocess.Popen([apache_ctl, "-V"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, _ = proc.communicate()
except OSError:
sys.exit(_NO_APACHECTL)
server_root = config_file = ""
for line in output.splitlines():
# Relevant output lines are of the form: -D DIRECTIVE="VALUE"
if "HTTPD_ROOT" in line:
server_root = line[line.find('"')+1:-1]
elif "SERVER_CONFIG_FILE" in line:
config_file = line[line.find('"')+1:-1]
if not (server_root and config_file):
sys.exit("Unable to locate Apache configuration. Please run this "
"script again and specify --server-root and --config-file")
return server_root, config_file
def get_args():
"""Parses command line arguments
:returns: Parsed command line options
:rtype: argparse.Namespace
"""
parser = argparse.ArgumentParser(description=_DESCRIPTION)
parser.add_argument("-c", "--apache-ctl", default="apachectl",
help="path to the `apachectl` binary")
parser.add_argument("-d", "--server-root",
help=("location of the root directory of your Apache "
"configuration"))
parser.add_argument("-f", "--config-file",
help=("location of your main Apache configuration "
"file relative to the server root"))
args = parser.parse_args()
# args.server_root XOR args.config_file
if bool(args.server_root) != bool(args.config_file):
sys.exit("If either --server-root and --config-file are specified, "
"they both must be included")
elif args.server_root and args.config_file:
args.server_root = os.path.abspath(args.server_root)
args.config_file = os.path.abspath(args.config_file)
if args.config_file.startswith(args.server_root):
args.config_file = args.config_file[len(args.server_root)+1:]
else:
sys.exit("This script expects the Apache configuration file to be "
"inside the server root")
return args
def main():
"""Main script execution"""
args = get_args()
if args.server_root is None:
args.server_root, args.config_file = locate_config(args.apache_ctl)
verify_config(args)
tempdir = setup_tempdir(args)
atexit.register(lambda: shutil.rmtree(tempdir))
make_and_verify_selection(args.server_root, tempdir)
tarpath = os.path.join(tempdir, "config.tar.gz")
# contextlib.closing used for py26 support
with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar:
tar.add(tempdir, arcname=".")
# TODO: Submit tarpath
if __name__ == "__main__":
main() # pragma: no cover

View file

@ -0,0 +1,234 @@
"""Tests for letshelp.letshelp_letsencrypt_apache.py"""
import argparse
import functools
import os
import pkg_resources
import subprocess
import tarfile
import tempfile
import unittest
import mock
import letshelp_letsencrypt.apache as letshelp_le_apache
_PARTIAL_CONF_PATH = os.path.join("mods-available", "ssl.load")
_PARTIAL_LINK_PATH = os.path.join("mods-enabled", "ssl.load")
_CONFIG_FILE = pkg_resources.resource_filename(
__name__, os.path.join("testdata", _PARTIAL_CONF_PATH))
_PASSWD_FILE = pkg_resources.resource_filename(
__name__, os.path.join("testdata", "uncommonly_named_p4sswd"))
_KEY_FILE = pkg_resources.resource_filename(
__name__, os.path.join("testdata", "uncommonly_named_k3y"))
_SECRET_FILE = pkg_resources.resource_filename(
__name__, os.path.join("testdata", "super_secret_file.txt"))
_MODULE_NAME = "letshelp_letsencrypt.apache"
_COMPILE_SETTINGS = """Server version: Apache/2.4.10 (Debian)
Server built: Mar 15 2015 09:51:43
Server's Module Magic Number: 20120211:37
Server loaded: APR 1.5.1, APR-UTIL 1.5.4
Compiled using: APR 1.5.1, APR-UTIL 1.5.4
Architecture: 64-bit
Server MPM: event
threaded: yes (fixed thread count)
forked: yes (variable process count)
Server compiled with....
-D APR_HAS_SENDFILE
-D APR_HAS_MMAP
-D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
-D APR_USE_SYSVSEM_SERIALIZE
-D APR_USE_PTHREAD_SERIALIZE
-D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-D APR_HAS_OTHER_CHILD
-D AP_HAVE_RELIABLE_PIPED_LOGS
-D DYNAMIC_MODULE_LIMIT=256
-D HTTPD_ROOT="/etc/apache2"
-D SUEXEC_BIN="/usr/lib/apache2/suexec"
-D DEFAULT_PIDLOG="/var/run/apache2.pid"
-D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
-D DEFAULT_ERRORLOG="logs/error_log"
-D AP_TYPES_CONFIG_FILE="mime.types"
-D SERVER_CONFIG_FILE="apache2.conf"
"""
class LetsHelpApacheTest(unittest.TestCase):
@mock.patch(_MODULE_NAME + ".copy_config")
def test_make_and_verify_selection(self, mock_copy_config):
mock_copy_config.return_value = (["apache2.conf"], ["apache2"])
with mock.patch("__builtin__.raw_input") as mock_input:
with mock.patch(_MODULE_NAME + ".sys.stdout"):
mock_input.side_effect = ["Yes", "No"]
letshelp_le_apache.make_and_verify_selection("root", "temp")
self.assertRaises(
SystemExit, letshelp_le_apache.make_and_verify_selection,
"server_root", "temp_dir")
def test_copy_config(self):
tempdir = tempfile.mkdtemp()
server_root = pkg_resources.resource_filename(__name__, "testdata")
letshelp_le_apache.copy_config(server_root, tempdir)
temp_testdata = os.path.join(tempdir, "testdata")
self.assertFalse(os.path.exists(os.path.join(
temp_testdata, os.path.basename(_PASSWD_FILE))))
self.assertFalse(os.path.exists(os.path.join(
temp_testdata, os.path.basename(_KEY_FILE))))
self.assertFalse(os.path.exists(os.path.join(
temp_testdata, os.path.basename(_SECRET_FILE))))
self.assertTrue(os.path.exists(os.path.join(
temp_testdata, _PARTIAL_CONF_PATH)))
self.assertTrue(os.path.exists(os.path.join(
temp_testdata, _PARTIAL_LINK_PATH)))
def test_copy_file_without_comments(self):
dest = tempfile.mkstemp()[1]
letshelp_le_apache.copy_file_without_comments(_PASSWD_FILE, dest)
with open(_PASSWD_FILE) as original:
with open(dest) as copy:
for original_line, copied_line in zip(original, copy):
self.assertEqual(original_line, copied_line)
@mock.patch(_MODULE_NAME + ".subprocess.Popen")
def test_safe_config_file(self, mock_popen):
mock_popen().communicate.return_value = ("PEM RSA private key", None)
self.assertFalse(letshelp_le_apache.safe_config_file("filename"))
mock_popen().communicate.return_value = ("ASCII text", None)
self.assertFalse(letshelp_le_apache.safe_config_file(_PASSWD_FILE))
self.assertFalse(letshelp_le_apache.safe_config_file(_KEY_FILE))
self.assertFalse(letshelp_le_apache.safe_config_file(_SECRET_FILE))
self.assertTrue(letshelp_le_apache.safe_config_file(_CONFIG_FILE))
@mock.patch(_MODULE_NAME + ".subprocess.Popen")
def test_tempdir(self, mock_popen):
mock_popen().communicate.side_effect = [
("version", None), ("modules", None), ("vhosts", None)]
args = _get_args()
tempdir = letshelp_le_apache.setup_tempdir(args)
with open(os.path.join(tempdir, "config_file")) as config_fd:
self.assertEqual(config_fd.read(), args.config_file + "\n")
with open(os.path.join(tempdir, "version")) as version_fd:
self.assertEqual(version_fd.read(), "version")
with open(os.path.join(tempdir, "modules")) as modules_fd:
self.assertEqual(modules_fd.read(), "modules")
with open(os.path.join(tempdir, "vhosts")) as vhosts_fd:
self.assertEqual(vhosts_fd.read(), "vhosts")
@mock.patch(_MODULE_NAME + ".subprocess.check_call")
def test_verify_config(self, mock_check_call):
args = _get_args()
mock_check_call.side_effect = [
None, OSError, subprocess.CalledProcessError(1, "apachectl")]
letshelp_le_apache.verify_config(args)
self.assertRaises(SystemExit, letshelp_le_apache.verify_config, args)
self.assertRaises(SystemExit, letshelp_le_apache.verify_config, args)
@mock.patch(_MODULE_NAME + ".subprocess.Popen")
def test_locate_config(self, mock_popen):
mock_popen().communicate.side_effect = [
OSError, ("bad_output", None), (_COMPILE_SETTINGS, None),]
self.assertRaises(
SystemExit, letshelp_le_apache.locate_config, "ctl")
self.assertRaises(
SystemExit, letshelp_le_apache.locate_config, "ctl")
server_root, config_file = letshelp_le_apache.locate_config("ctl")
self.assertEqual(server_root, "/etc/apache2")
self.assertEqual(config_file, "apache2.conf")
@mock.patch(_MODULE_NAME + ".argparse")
def test_get_args(self, mock_argparse):
argv = ["-d", "/etc/apache2"]
mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
self.assertRaises(SystemExit, letshelp_le_apache.get_args)
server_root = "/etc/apache2"
config_file = server_root + "/apache2.conf"
argv = ["-d", server_root, "-f", config_file]
mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
args = letshelp_le_apache.get_args()
self.assertEqual(args.apache_ctl, "apachectl")
self.assertEqual(args.server_root, server_root)
self.assertEqual(args.config_file, os.path.basename(config_file))
server_root = "/etc/apache2"
config_file = "/etc/httpd/httpd.conf"
argv = ["-d", server_root, "-f", config_file]
mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
self.assertRaises(SystemExit, letshelp_le_apache.get_args)
def test_main_with_args(self):
with mock.patch(_MODULE_NAME + ".get_args"):
self._test_main_common()
def test_main_without_args(self):
with mock.patch(_MODULE_NAME + ".get_args") as get_args:
args = _get_args()
server_root, config_file = args.server_root, args.config_file
args.server_root = args.config_file = None
get_args.return_value = args
with mock.patch(_MODULE_NAME + ".locate_config") as locate:
locate.return_value = (server_root, config_file)
self._test_main_common()
def _test_main_common(self):
with mock.patch(_MODULE_NAME + ".verify_config"):
with mock.patch(_MODULE_NAME + ".setup_tempdir") as mock_setup:
tempdir_path = tempfile.mkdtemp()
mock_setup.return_value = tempdir_path
with mock.patch(_MODULE_NAME + ".make_and_verify_selection"):
testdir_basename = "test"
os.mkdir(os.path.join(tempdir_path, testdir_basename))
letshelp_le_apache.main()
tar = tarfile.open(os.path.join(
tempdir_path, "config.tar.gz"))
tempdir = tar.next()
self.assertTrue(tempdir.isdir())
self.assertEqual(tempdir.name, ".")
testdir = tar.next()
self.assertTrue(testdir.isdir())
self.assertEqual(os.path.basename(testdir.name),
testdir_basename)
self.assertEqual(tar.next(), None)
def _create_mock_parser(argv):
parser = argparse.ArgumentParser()
mock_parser = mock.MagicMock()
mock_parser.add_argument = parser.add_argument
mock_parser.parse_args = functools.partial(parser.parse_args, argv)
return mock_parser
def _get_args():
args = argparse.Namespace()
args.apache_ctl = "apache_ctl"
args.config_file = "config_file"
args.server_root = "server_root"
return args
if __name__ == "__main__":
unittest.main() # pragma: no cover

View file

@ -0,0 +1,2 @@
# Depends: setenvif mime socache_shmcb
LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so

View file

@ -0,0 +1 @@
../mods-available/ssl.load

View file

@ -0,0 +1 @@
hunter2

View file

@ -0,0 +1,6 @@
-----BEGIN RSA PRIVATE KEY-----
MIGrAgEAAiEAm2Fylv+Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEkCAwEAAQIh
AJT0BA/xD01dFCAXzSNyj9nfSZa3NpqzJZZn/eOm7vghAhEAzUVNZn4lLLBD1R6N
E8TKNQIRAMHHyn3O5JeY36lwKwkUlEUCEAliRauN0L0+QZuYjfJ9aJECEGx4dru3
rTPCyighdqWNlHUCEQCiLjlwSRtWgmMBudCkVjzt
-----END RSA PRIVATE KEY-----

View file

@ -0,0 +1 @@
johntheripper:$apr1$fIGE9.JL$jTCwNWZy9Ak/yvOLuOyzQ1

View file

@ -0,0 +1,23 @@
import sys
from setuptools import setup
from setuptools import find_packages
install_requires = []
if sys.version_info < (2, 7):
install_requires.append("mock<1.1.0")
else:
install_requires.append("mock")
setup(
name="letshelp-letsencrypt",
packages=find_packages(),
install_requires=install_requires,
entry_points={
'console_scripts': [
"letshelp-letsencrypt-apache = letshelp_letsencrypt.apache:main",
],
},
include_package_data=True,
)

View file

@ -23,4 +23,5 @@ rm -f .coverage # --cover-erase is off, make sure stats are correct
cover letsencrypt 97 && \
cover acme 100 && \
cover letsencrypt_apache 100 && \
cover letsencrypt_nginx 96
cover letsencrypt_nginx 96 && \
cover letshelp_letsencrypt 100

View file

@ -10,12 +10,13 @@ envlist = py26,py27,cover,lint
[testenv]
commands =
pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx
pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt
# -q does not suppress errors
python setup.py test -q
python setup.py test -q -s acme
python setup.py test -q -s letsencrypt_apache
python setup.py test -q -s letsencrypt_nginx
python setup.py test -q -s letshelp_letsencrypt
setenv =
PYTHONPATH = {toxinidir}
@ -25,7 +26,7 @@ setenv =
[testenv:cover]
basepython = python2.7
commands =
pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx
pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt
./tox.cover.sh
[testenv:lint]
@ -35,9 +36,10 @@ basepython = python2.7
# duplicate code checking; if one of the commands fails, others will
# continue, but tox return code will reflect previous error
commands =
pip install -r requirements.txt -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test
pip install -r requirements.txt -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt
pylint --rcfile=.pylintrc letsencrypt
pylint --rcfile=.pylintrc acme/acme
pylint --rcfile=.pylintrc letsencrypt-apache/letsencrypt_apache
pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx
pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test
pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt