Add again some bash scripts to avoid breaking to much retro-compatiblity on third party scripts

certbot-nginx/tests/boulder-integration.conf.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Based on
+# https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/
+# https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf
+
+# USAGE: ./boulder-integration.conf.sh /path/to/root cert.key cert.pem >> nginx.conf
+
+ROOT=$1
+CERT_KEY_PATH=$2
+CERT_PATH=$3
+
+cat <<EOF
+# This error log will be written regardless of server scope error_log
+# definitions, so we have to set this here in the main scope.
+#
+# Even doing this, Nginx will still try to create the default error file, and
+# log a non-fatal error when it fails. After that things will work, however.
+error_log $ROOT/error.log;
+
+# The pidfile will be written to /var/run unless this is set.
+pid $ROOT/nginx.pid;
+
+worker_processes 1;
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  # Set an array of temp, cache and log file options that will otherwise default to
+  # restricted locations accessible only to root.
+  client_body_temp_path $ROOT/client_body;
+  fastcgi_temp_path $ROOT/fastcgi_temp;
+  proxy_temp_path $ROOT/proxy_temp;
+  #scgi_temp_path $ROOT/scgi_temp;
+  #uwsgi_temp_path $ROOT/uwsgi_temp;
+  access_log $ROOT/error.log;
+
+  # This should be turned off in a Virtualbox VM, as it can cause some
+  # interesting issues with data corruption in delivered files.
+  sendfile off;
+
+  tcp_nopush on;
+  tcp_nodelay on;
+  keepalive_timeout 65;
+  types_hash_max_size 2048;
+
+  #include /etc/nginx/mime.types;
+  index index.html index.htm index.php;
+
+  log_format   main '\$remote_addr - \$remote_user [\$time_local] \$status '
+    '"\$request" \$body_bytes_sent "\$http_referer" '
+    '"\$http_user_agent" "\$http_x_forwarded_for"';
+
+  default_type application/octet-stream;
+
+  server {
+    # IPv4.
+    listen 5002 $default_server;
+    # IPv6.
+    listen [::]:5002 $default_server;
+    server_name nginx.wtf nginx2.wtf;
+
+    root $ROOT/webroot;
+
+    location / {
+      # First attempt to serve request as file, then as directory, then fall
+      # back to index.html.
+      try_files \$uri \$uri/ /index.html;
+    }
+  }
+
+  server {
+    listen 5002;
+    listen [::]:5002;
+    server_name nginx3.wtf;
+
+    root $ROOT/webroot;
+
+    location /.well-known/ {
+      return 404;
+    }
+
+    return 301 https://\$host\$request_uri;
+  }
+
+  server {
+    listen 8082;
+    listen [::]:8082;
+    server_name nginx4.wtf nginx5.wtf;
+  }
+
+  server {
+    listen 5002;
+    listen [::]:5002;
+    listen 5001 ssl;
+    listen [::]:5001 ssl;
+    if (\$scheme != "https") {
+      return 301 https://\$host\$request_uri;
+    }
+    server_name nginx6.wtf nginx7.wtf;
+
+    ssl_certificate ${CERT_PATH};
+    ssl_certificate_key ${CERT_KEY_PATH};
+  }
+}
+EOF

tests/boulder-fetch.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Download and run Boulder instance for integration testing
+set -xe
+
+# Clone Boulder into a GOPATH-style directory structure even if Go isn't
+# installed, because Boulder's docker-compose.yml file wll look for it there.
+export GOPATH=${GOPATH:-$HOME/gopath}
+BOULDERPATH=${BOULDERPATH:-$GOPATH/src/github.com/letsencrypt/boulder}
+if [ ! -d ${BOULDERPATH} ]; then
+  git clone --depth=1 https://github.com/letsencrypt/boulder ${BOULDERPATH}
+fi
+
+cd ${BOULDERPATH}
+
+docker-compose up -d boulder
+
+set +x  # reduce verbosity while waiting for boulder
+for n in `seq 1 150` ; do
+  if curl http://localhost:4000/directory 2>/dev/null; then
+    break
+  else
+    sleep 1
+  fi
+done
+
+if ! curl http://localhost:4000/directory 2>/dev/null; then
+  echo "timed out waiting for boulder to start"
+  exit 1
+fi
+
+# Setup the DNS resolution used by boulder instance to docker host
+curl -X POST -d '{"ip":"10.77.77.1"}' http://localhost:8055/set-default-ipv4

tests/integration/_common.sh

@@ -0,0 +1,74 @@
+# The -t is required on macOS. It provides a template file path for
+# the kernel to use.
+root=${root:-$(mktemp -d -t leitXXXX)}
+echo "Root integration tests directory: $root"
+config_dir="$root/conf"
+https_port=5001
+http_01_port=5002
+sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')"
+export root config_dir https_port http_01_port sources
+certbot_path="$(command -v certbot)"
+# Flags that are added here will be added to Certbot calls within
+# certbot_test_no_force_renew.
+other_flags="--config-dir $config_dir --work-dir $root/work"
+other_flags="$other_flags --logs-dir $root/logs"
+
+certbot_test () {
+    certbot_test_no_force_renew \
+        --renew-by-default \
+        "$@"
+}
+
+# Succeeds if Certbot version is at least the given version number and fails
+# otherwise. This is useful for making sure Certbot has certain features
+# available. The patch version is currently ignored.
+#
+# Arguments:
+#   First argument is the minimum major version
+#   Second argument is the minimum minor version
+version_at_least () {
+    # Certbot major and minor version (e.g. 0.30)
+    major_minor=$("$certbot_path" --version 2>&1 | cut -d' ' -f2 | cut -d. -f1,2)
+    major=$(echo "$major_minor" | cut -d. -f1)
+    minor=$(echo "$major_minor" | cut -d. -f2)
+    # Test that either the major version is greater or major version is equal
+    # and minor version is greater than or equal to.
+    [ \( "$major" -gt "$1" \) -o \( "$major" -eq "$1" -a "$minor" -ge "$2" \) ]
+}
+
+# Use local ACMEv2 endpoint if requested and SERVER isn't already set.
+if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then
+    SERVER="http://localhost:4001/directory"
+fi
+
+# --no-random-sleep-on-renew was added in
+# https://github.com/certbot/certbot/pull/6599 and first released in Certbot
+# 0.30.0.
+if version_at_least 0 30; then
+  other_flags="$other_flags --no-random-sleep-on-renew"
+fi
+
+certbot_test_no_force_renew () {
+    omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*"
+    omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*"
+    omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/"
+    omit_patterns="$omit_patterns,certbot-nginx/certbot_nginx/parser_obj.py"
+    coverage run \
+        --append \
+        --source $sources \
+        --omit $omit_patterns \
+        "$certbot_path" \
+            --server "${SERVER:-http://localhost:4000/directory}" \
+            --no-verify-ssl \
+            --http-01-port $http_01_port \
+            --https-port $https_port \
+            --manual-public-ip-logging-ok \
+            $other_flags \
+            --non-interactive \
+            --no-redirect \
+            --agree-tos \
+            --register-unsafely-without-email \
+            --debug \
+            -vv \
+            "$@"
+}

tests/lock_test.py

@@ -108,16 +108,13 @@ def set_up_nginx_dir(root_path):
     """
     # Get the root of the git repository
     repo_root = check_call('git rev-parse --show-toplevel'.split()).strip()
-    # We add manually nginx_config module, because certbot_integration_tests may not be installed.
-    conf_path = os.path.join(repo_root, 'certbot-ci', 'certbot_integration_tests', 'nginx_tests')
-    sys.path.append(conf_path)
-    import nginx_config  # pylint: disable=import-error
+    conf_script = os.path.join(
+        repo_root, 'certbot-nginx', 'tests', 'boulder-integration.conf.sh')
+    # Prepare self-signed certificates for Nginx
     key_path, cert_path = setup_certificate(root_path)
-    config = nginx_config.construct_nginx_config(root_path, os.path.join(root_path, 'webroot'),
-                                                 5002, 5001, 8082, False,
-                                                 key_path=key_path, cert_path=cert_path)
+    # Generate Nginx configuration
     with open(os.path.join(root_path, 'nginx.conf'), 'w') as f:
-        f.write(config)
+        f.write(check_call(['/bin/sh', conf_script, root_path, key_path, cert_path]))
 
 
 def set_up_command(config_dir, logs_dir, work_dir, nginx_dir):
@@ -188,6 +185,7 @@ def setup_certificate(workspace):
 
     return key_path, cert_path
 
+
 def test_command(command, directories):
     """Assert Certbot acquires locks in a specific order.
 

tox.ini

@@ -155,7 +155,7 @@ passenv =
 
 [testenv:apacheconftest-with-pebble]
 commands =
-    {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/pebble-fetch-start.sh
+    {toxinidir}/tests/pebble-fetch.sh
     {[testenv:apacheconftest]commands}
 passenv =
     HOME