Merge remote-tracking branch 'origin/master' into apache-conf-test

This commit is contained in:
Peter Eckersley 2015-12-22 15:57:20 -08:00
commit c77a6fa674
17 changed files with 331 additions and 30 deletions

View file

@ -32,6 +32,35 @@ if apt-cache show python-virtualenv > /dev/null ; then
virtualenv="$virtualenv python-virtualenv"
fi
augeas_pkg=libaugeas0
AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2`
if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then
if lsb_release -a | grep -q wheezy ; then
if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then
# This can theoretically error if sources.list.d is empty, but in that case we don't care.
if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then
/bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..."
sleep 1s
/bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..."
sleep 1s
/bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..."
sleep 1s
/bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")'
echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list
apt-get update
fi
fi
apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0
augeas_pkg=
else
echo "No libaugeas0 version is available that's new enough to run the"
echo "Let's Encrypt apache plugin..."
fi
# XXX add a case for ubuntu PPAs
fi
apt-get install -y --no-install-recommends \
git \
python \
@ -39,11 +68,13 @@ apt-get install -y --no-install-recommends \
$virtualenv \
gcc \
dialog \
libaugeas0 \
$augeas_pkg \
libssl-dev \
libffi-dev \
ca-certificates \
if ! command -v virtualenv > /dev/null ; then
echo Failed to install a working \"virtualenv\" command, exiting
exit 1

View file

@ -64,7 +64,7 @@ or for full help, type:
``letsencrypt-auto`` is the recommended method of running the Let's Encrypt
client beta releases on systems that don't have a packaged version. Debian,
Arch linux and FreeBSD now have native packages, so on those
Arch linux, FreeBSD, and OpenBSD now have native packages, so on those
systems you can just install ``letsencrypt`` (and perhaps
``letsencrypt-apache``). If you'd like to run the latest copy from Git, or
run your own locally modified copy of the client, follow the instructions in
@ -351,6 +351,11 @@ Operating System Packages
* Port: ``cd /usr/ports/security/py-letsencrypt && make install clean``
* Package: ``pkg install py27-letsencrypt``
**OpenBSD**
* Port: ``cd /usr/ports/security/letsencrypt/client && make install clean``
* Package: ``pkg_add letsencrypt``
**Arch Linux**
.. code-block:: shell

View file

@ -59,7 +59,7 @@ let empty = Util.empty_dos
let indent = Util.indent
(* borrowed from shellvars.aug *)
let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ '"\t\r\n])|\\\\"|\\\\'/
let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/
let char_arg_sec = /[^ '"\t\r\n>]|\\\\"|\\\\'/
let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/

View file

@ -542,7 +542,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
"""
if "ssl_module" not in self.parser.modules:
self.enable_mod("ssl", temp=temp)
if self.version >= (2, 4) and "socache_shmcb_module" not in self.parser.modules:
self.enable_mod("socache_shmcb", temp=temp)
# Check for Listen <port>
# Note: This could be made to also look for ip:443 combo
listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")]
@ -1399,7 +1400,7 @@ def _get_mod_deps(mod_name):
"""
deps = {
"ssl": ["setenvif", "mime", "socache_shmcb"]
"ssl": ["setenvif", "mime"]
}
return deps.get(mod_name, [])

View file

@ -19,7 +19,6 @@ class ApacheParser(object):
:ivar str root: Normalized absolute path to the server root
directory. Without trailing slash.
:ivar str root: Server root
:ivar set modules: All module names that are currently enabled.
:ivar dict loc: Location to place directives, root - configuration origin,
default - user config file, name - NameVirtualHost,
@ -36,6 +35,7 @@ class ApacheParser(object):
# https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine
# This only handles invocation parameters and Define directives!
self.variables = {}
self.unparsable = False
self.update_runtime_variables(ctl)
self.aug = aug
@ -60,6 +60,11 @@ class ApacheParser(object):
# Sites-available is not included naturally in configuration
self._parse_file(os.path.join(self.root, "sites-available") + "/*")
#check to see if there were unparsed define statements
if self.unparsable:
if self.find_dir("Define", exclude=False):
raise errors.PluginError("Error parsing runtime variables")
def init_modules(self):
"""Iterates on the configuration until no new modules are loaded.
@ -101,7 +106,8 @@ class ApacheParser(object):
try:
matches.remove("DUMP_RUN_CFG")
except ValueError:
raise errors.PluginError("Unable to parse runtime variables")
self.unparsable = True
return
for match in matches:
if match.count("=") > 1:

View file

@ -28,10 +28,21 @@ class TwoVhost80Test(util.ApacheTest):
self.config = util.get_apache_configurator(
self.config_path, self.config_dir, self.work_dir)
self.config = self.mock_deploy_cert(self.config)
self.vh_truth = util.get_vh_truth(
self.temp_dir, "debian_apache_2_4/two_vhost_80")
def mock_deploy_cert(self, config):
"""A test for a mock deploy cert"""
self.config.real_deploy_cert = self.config.deploy_cert
def mocked_deploy_cert(*args, **kwargs):
"""a helper to mock a deployed cert"""
with mock.patch(
"letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"):
config.real_deploy_cert(*args, **kwargs)
self.config.deploy_cert = mocked_deploy_cert
return self.config
def tearDown(self):
shutil.rmtree(self.temp_dir)
shutil.rmtree(self.config_dir)
@ -251,6 +262,7 @@ class TwoVhost80Test(util.ApacheTest):
# Get the default 443 vhost
self.config.assoc["random.demo"] = self.vh_truth[1]
self.config = self.mock_deploy_cert(self.config)
self.config.deploy_cert(
"random.demo", "example/cert.pem", "example/key.pem",
"example/cert_chain.pem", "example/fullchain.pem")
@ -277,6 +289,7 @@ class TwoVhost80Test(util.ApacheTest):
def test_deploy_cert_newssl_no_fullchain(self):
self.config = util.get_apache_configurator(
self.config_path, self.config_dir, self.work_dir, version=(2, 4, 16))
self.config = self.mock_deploy_cert(self.config)
self.config.parser.modules.add("ssl_module")
self.config.parser.modules.add("mod_ssl.c")
@ -290,6 +303,7 @@ class TwoVhost80Test(util.ApacheTest):
def test_deploy_cert_old_apache_no_chain(self):
self.config = util.get_apache_configurator(
self.config_path, self.config_dir, self.work_dir, version=(2, 4, 7))
self.config = self.mock_deploy_cert(self.config)
self.config.parser.modules.add("ssl_module")
self.config.parser.modules.add("mod_ssl.c")

View file

@ -151,8 +151,8 @@ class BasicParserTest(util.ParserTest):
@mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
def test_update_runtime_vars_bad_output(self, mock_cfg):
mock_cfg.return_value = "Define: TLS=443=24"
self.assertRaises(
errors.PluginError, self.parser.update_runtime_variables, "ctl")
self.parser.update_runtime_variables("ctl")
self.assertTrue(self.parser.unparsable)
mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24"
self.assertRaises(
@ -185,6 +185,21 @@ class ParserInitTest(util.ApacheTest):
shutil.rmtree(self.config_dir)
shutil.rmtree(self.work_dir)
@mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
def test_unparsable(self, mock_cfg):
from letsencrypt_apache.parser import ApacheParser
def unparsable_true(self, arg):
"""a helper to set the self unparsabale to true"""
print "side effect has passed in arg: %s", arg
self.unparsable = True
with mock.patch.object(ApacheParser, 'update_runtime_variables', autospec=True) as urv:
urv.side_effect = unparsable_true
mock_cfg.return_value = ('Define: TEST')
self.assertRaises(
errors.PluginError,
ApacheParser, self.aug, os.path.relpath(self.config_path), "ctl")
self.assertEquals(1, 1)
def test_root_normalized(self):
from letsencrypt_apache.parser import ApacheParser

View file

@ -78,7 +78,9 @@ class TlsSniPerformTest(util.ApacheTest):
# pylint: disable=protected-access
self.sni._setup_challenge_cert = mock_setup_cert
sni_responses = self.sni.perform()
with mock.patch(
"letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"):
sni_responses = self.sni.perform()
self.assertEqual(mock_setup_cert.call_count, 2)

View file

@ -97,6 +97,7 @@ DeterminePythonVersion() {
export LE_PYTHON=${LE_PYTHON:-python}
else
echo "Cannot find any Pythons... please install one!"
exit 1
fi
PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`

View file

@ -66,8 +66,16 @@ class AuthenticatorTest(unittest.TestCase):
def test_prepare_reraises_other_errors(self):
self.auth.full_path = os.path.join(self.path, "null")
permission_canary = os.path.join(self.path, "rnd")
with open(permission_canary, "w") as f:
f.write("thingimy")
os.chmod(self.path, 0o000)
self.assertRaises(errors.PluginError, self.auth.prepare)
try:
open(permission_canary, "r")
print "Warning, running tests as root skips permissions tests..."
except IOError:
# ok, permissions work, test away...
self.assertRaises(errors.PluginError, self.auth.prepare)
os.chmod(self.path, 0o700)
@mock.patch("letsencrypt.plugins.webroot.os.chown")

View file

@ -181,7 +181,9 @@ def main(cli_args=sys.argv[1:]):
# RenewableCert object for this cert at all, which could
# dramatically improve performance for large deployments
# where autorenewal is widely turned off.
cert = storage.RenewableCert(renewal_file, cli_config)
cert = storage.RenewableCert(
os.path.join(cli_config.renewal_configs_dir, renewal_file),
cli_config)
except errors.CertStorageError:
# This indicates an invalid renewal configuration file, such
# as one missing a required parameter (in the future, perhaps

View file

@ -260,7 +260,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes
:returns: The path to the current version of the specified
member.
:rtype: str
:rtype: str or None
"""
if kind not in ALL_FOUR:

View file

@ -764,6 +764,8 @@ class RenewableCertTests(BaseRenewableCertTest):
def test_bad_config_file(self):
from letsencrypt import renewer
os.unlink(os.path.join(self.cli_config.renewal_configs_dir,
"example.org.conf"))
with open(os.path.join(self.cli_config.renewal_configs_dir,
"bad.conf"), "w") as f:
f.write("incomplete = configfile\n")

View file

@ -0,0 +1,21 @@
<VirtualHost *:80>
WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=_graphite group=_graphite
WSGIProcessGroup _graphite
WSGIImportScript /usr/share/graphite-web/graphite.wsgi process-group=_graphite application-group=%{GLOBAL}
WSGIScriptAlias / /usr/share/graphite-web/graphite.wsgi
Alias /content/ /usr/share/graphite-web/static/
<Location "/content/">
SetHandler None
</Location>
ErrorLog ${APACHE_LOG_DIR}/graphite-web_error.log
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/graphite-web_access.log combined
</VirtualHost>

View file

@ -0,0 +1,7 @@
<IfModule mod_rewrite.c>
RewriteEngine On
RewriteCond %{REQUEST_URI} ^.*(,|;|:|<|>|">|"<|/|\\\.\.\\).* [NC,OR]
RewriteCond %{REQUEST_URI} ^.*(\=|\@|\[|\]|\^|\`|\{|\}|\~).* [NC,OR]
RewriteCond %{REQUEST_URI} ^.*(\'|%0A|%0D|%27|%3C|%3E|%00).* [NC]
RewriteRule ^(.*)$ - [F,L]
</IfModule>

120
tools/half-sign.c Normal file
View file

@ -0,0 +1,120 @@
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <openssl/evp.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>
// This program can be used to perform RSA public key signatures given only
// the hash of the file to be signed as input.
// Sign with SHA1
#define HASH_SIZE 20
void usage() {
printf("half-sign <private key file> [binary hash file]\n");
printf("\n");
printf(" Computes and prints a binary RSA signature over data given the SHA1 hash of\n");
printf(" the data as input.\n");
printf("\n");
printf(" <private key file> should be PEM encoded.\n");
printf("\n");
printf(" The input SHA1 hash should be %d bytes in length. If no binary hash file is\n", HASH_SIZE);
printf(" specified, it will be read from stdin.\n");
exit(1);
}
void sign_hashed_data(EVP_PKEY *signing_key, unsigned char *md, size_t mdlen) {
// cribbed from the openssl EVP_PKEY_sign man page
EVP_PKEY_CTX *ctx;
unsigned char *sig;
size_t siglen;
/* NB: assumes signing_key, md and mdlen are already set up
* and that signing_key is an RSA private key
*/
ctx = EVP_PKEY_CTX_new(signing_key, NULL);
if ((!ctx)
|| (EVP_PKEY_sign_init(ctx) <= 0)
|| (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0)
|| (EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha1()) <= 0)) {
fprintf(stderr, "Failure establishing ctx for signature\n");
exit(1);
}
/* Determine buffer length */
if (EVP_PKEY_sign(ctx, NULL, &siglen, md, mdlen) <= 0) {
fprintf(stderr, "Unable to determine buffer length for signature\n");
exit(1);
}
sig = OPENSSL_malloc(siglen);
if (!sig) {
fprintf(stderr, "Malloc failed\n");
exit(1);
}
if (EVP_PKEY_sign(ctx, sig, &siglen, md, mdlen) <= 0) {
fprintf(stderr, "Signature error\n");
exit(1);
}
/* Signature is siglen bytes written to buffer sig */
fwrite(sig, siglen, 1, stdout);
}
EVP_PKEY *read_private_key(char *filename) {
FILE *keyfile;
EVP_PKEY *privkey;
keyfile = fopen(filename, "r");
if (!keyfile) {
fprintf(stderr, "Failed to open private key.pem file %s\n", filename);
exit(1);
}
privkey = PEM_read_PrivateKey(keyfile, NULL, NULL, NULL);
if (!privkey) {
fprintf(stderr, "Failed to read PEM private key from %s\n", filename);
exit(1);
}
if (EVP_PKEY_type(privkey->type) != EVP_PKEY_RSA) {
fprintf(stderr, "%s was a non-RSA key\n", filename);
exit(1);
}
return privkey;
}
int main(int argc, char *argv[]) {
FILE *input;
unsigned char *buffer;
int test;
EVP_PKEY *privkey;
if (argc > 3 || argc < 2)
usage();
if (argc < 3 || strcmp(argv[2],"-") == 0)
input = stdin;
else {
input = fopen(argv[2], "r");
if (!input) usage();
}
privkey = read_private_key(argv[1]);
buffer = malloc(HASH_SIZE);
if (!buffer) {
fprintf(stderr, "Argh, malloc failed\n");
exit(1);
}
if (fread(buffer, HASH_SIZE, 1, input) != 1) {
perror("half-sign: Failed to read SHA1 from input\n");
exit(1);
}
test = fgetc(input);
if (test != EOF && test != '\n') {
fprintf(stderr,"Error, more than %d bytes fed to half-sign\n", HASH_SIZE);
fprintf(stderr,"Last byte was :%d\n" , (int) test);
exit(1);
}
sign_hashed_data(privkey, buffer, HASH_SIZE);
return 0;
}

View file

@ -1,13 +1,43 @@
#!/bin/sh -xe
#!/bin/bash -xe
# Release dev packages to PyPI
Usage() {
echo Usage:
echo "$0 [ --production ]"
exit 1
}
if [ "`dirname $0`" != "tools" ] ; then
echo Please run this script from the repo root
exit 1
fi
CheckVersion() {
# Args: <description of version type> <version number>
if ! echo "$2" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then
echo "$1 doesn't look like 1.2.3"
exit 1
fi
}
if [ "$1" = "--production" ] ; then
version="$2"
CheckVersion Version "$version"
echo Releasing production version "$version"...
nextversion="$3"
CheckVersion "Next version" "$nextversion"
RELEASE_BRANCH="candidate-$version"
else
version=`grep "__version__" letsencrypt/__init__.py | cut -d\' -f2 | sed s/\.dev0//`
version="$version.dev$(date +%Y%m%d)1"
RELEASE_BRANCH="dev-release"
echo Releasing developer version "$version"...
fi
RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2}
# Needed to fix problems with git signatures and pinentry
export GPG_TTY=$(tty)
version="0.0.0.dev$(date +%Y%m%d)"
DEV_RELEASE_BRANCH="dev-release"
RELEASE_GPG_KEY=A2CFB51FA275A7286234E7B24D17C995CD9775F2
# port for a local Python Package Index (used in testing)
PORT=${PORT:-1234}
@ -36,21 +66,29 @@ pip install -U wheel # setup.py bdist_wheel
# from current env when creating a child env
pip install -U virtualenv
root="$(mktemp -d -t le.$version.XXX)"
root_without_le="$version.$$"
root="./releases/le.$root_without_le"
echo "Cloning into fresh copy at $root" # clean repo = no artificats
git clone . $root
git rev-parse HEAD
cd $root
git branch -f "$DEV_RELEASE_BRANCH"
git checkout "$DEV_RELEASE_BRANCH"
if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then
git branch -f "$RELEASE_BRANCH"
fi
git checkout "$RELEASE_BRANCH"
for pkg_dir in $SUBPKGS
do
sed -i $x "s/^version.*/version = '$version'/" $pkg_dir/setup.py
done
sed -i "s/^__version.*/__version__ = '$version'/" letsencrypt/__init__.py
SetVersion() {
ver="$1"
for pkg_dir in $SUBPKGS
do
sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py
done
sed -i "s/^__version.*/__version__ = '$ver'/" letsencrypt/__init__.py
git add -p # interactive user input
git add -p $SUBPKGS # interactive user input
}
SetVersion "$version"
git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"
git tag --local-user "$RELEASE_GPG_KEY" \
--sign --message "Release $version" "$tag"
@ -68,7 +106,7 @@ do
echo "Signing ($pkg_dir)"
for x in dist/*.tar.gz dist/*.whl
do
gpg2 --detach-sign --armor --sign $x
gpg -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign $x
done
cd -
@ -97,16 +135,44 @@ pip install \
letsencrypt $SUBPKGS
# stop local PyPI
kill $!
cd ~-
# freeze before installing anything else, so that we know end-user KGS
# make sure "twine upload" doesn't catch "kgs"
if [ -d ../kgs ] ; then
echo Deleting old kgs...
rm -rf ../kgs
fi
mkdir ../kgs
kgs="../kgs/$version"
pip freeze | tee $kgs
pip install nose
nosetests letsencrypt $subpkgs_modules
for module in letsencrypt $subpkgs_modules ; do
echo testing $module
nosetests $module
done
deactivate
cd ..
echo Now in $PWD
name=${root_without_le%.*}
ext="${root_without_le##*.}"
rev="$(git rev-parse --short HEAD)"
echo tar cJvf $name.$rev.tar.xz $name.$rev
echo gpg -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz
cd ~-
echo "New root: $root"
echo "KGS is at $root/kgs"
echo "Test commands (in the letstest repo):"
echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta'
echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1'
echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh'
echo "In order to upload packages run the following command:"
echo twine upload "$root/dist.$version/*/*"
if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then
SetVersion "$nextversion".dev0
git diff
git commit -m "Bump version to $nextversion"
fi