Merge pull request #8086 from certbot/core20-squashed

Upgrade snap to be based on core20

This PR makes several changes to be built on top of the core20 base snap. Fixes #7854.

The main changes are to `snapcraft.yaml`. With Snapcraft 4.0/core20 base, the python plugin is a thin wrapper, basically creating a `venv` and installing the packages from the source. The trouble with this is that it runs pycache, creating caches that conflict from the different parts. So to solve that, we put everything in a single part. Other changes include:

- We use classic confinement, so we need to specify a bunch of python packages to `stage-packages`, as mentioned [here](https://forum.snapcraft.io/t/trouble-bundling-python-with-classic-confinement-in-core20-4-0-4/18234/2).
- The certbot executable now lives in `bin`, so specify running `certbot/bin`.
- Since `python-augeas` is now being pulled into the single part, remove the pinning from constraints so we can use the latest version directly from github.
- Precompile our `cryptography` and `cffi` wheels to be based on python3.8.

Separately, we had to upgrade the snapcraft docker image to be based on focal, due to the thin wrapper situation. This was accomplished [here](https://github.com/adferrand/snapcraft/pull/1).

snap/local/build_and_install.sh

@@ -21,7 +21,8 @@ source "${DIR}/common.sh"
 RegisterQemuHandlers
 ResolveArch "${SNAP_ARCH}"
 
-tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > snap-constraints.txt
+tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt \
+  | grep -v python-augeas > snap-constraints.txt
 
 pushd "${DIR}/packages"
 "${CERTBOT_DIR}/tools/simple_http_server.py" 8080 >/dev/null 2>&1 &

snap/local/compile_native_wheels.sh

@@ -14,7 +14,8 @@ source "${DIR}/common.sh"
 
 RegisterQemuHandlers
 
-tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > "${DIR}/snap-constraints.txt"
+tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt \
+  | grep -v python-augeas > "${DIR}/snap-constraints.txt"
 for SNAP_ARCH in ${TARGET_ARCHS}; do
     ResolveArch "${SNAP_ARCH}"
     DownloadQemuStatic "${QEMU_ARCH}" "${DIR}"
@@ -24,7 +25,7 @@ for SNAP_ARCH in ${TARGET_ARCHS}; do
         -v "${DIR}/qemu-${QEMU_ARCH}-static:/usr/bin/qemu-${QEMU_ARCH}-static" \
         -v "${DIR}:/workspace" \
         -w "/workspace" \
-        "${DOCKER_ARCH}/ubuntu:18.04" \
+        "${DOCKER_ARCH}/ubuntu:20.04" \
         sh -c "\
    apt-get update \
 && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-venv python3-dev libffi-dev libssl-dev gcc \

snap/snapcraft.yaml

@@ -15,7 +15,7 @@ description: |
    - Help you revoke the certificate if that ever becomes necessary.
 confinement: classic
 grade: devel
-base: core18
+base: core20
 adopt-info: certbot
 
 apps:
@@ -26,7 +26,7 @@ apps:
       AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist"
       LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
   renew:
-    command: certbot -q renew
+    command: bin/certbot -q renew
     daemon: oneshot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
@@ -35,58 +35,42 @@ apps:
     # Run approximately twice a day with randomization
     timer: 00:00~24:00/2
 
+
 parts:
-  python-augeas:
-    plugin: python
-    source: git://github.com/basak/python-augeas
-    source-branch: snap
-    python-version: python3
-    build-packages: [libaugeas-dev]
-  acme:
-    plugin: python
-    source: .
-    source-subdir: acme
-    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
-    python-version: python3
-    # To build cryptography and cffi if needed
-    build-packages: [libffi-dev, libssl-dev]
   certbot:
     plugin: python
     source: .
-    source-subdir: certbot
     constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
-    python-version: python3
-    after: [acme]
+    python-packages:
+      - git+https://github.com/basak/python-augeas.git@snap
+      - ./acme
+      - ./certbot
+      - ./certbot-apache
+      - ./certbot-nginx
+    stage:
+      - -usr/lib/python3.8/sitecustomize.py # maybe unnecessary
+      # Prefer cffi
+      - -lib/python3.8/site-packages/augeas.py
+    stage-packages:
+      - libaugeas0
+      # added to stage python:
+      - libpython3-stdlib
+      - libpython3.8-stdlib
+      - libpython3.8-minimal
+      - python3-pip
+      - python3-setuptools
+      - python3-wheel
+      - python3-venv
+      - python3-minimal
+      - python3-distutils
+      - python3-pkg-resources
+      - python3.8-minimal
+    # To build cryptography and cffi if needed
+    build-packages: [libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
     override-pull: |
         snapcraftctl pull
-        snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//`
-    # Workaround for lack of site-packages leading to empty sitecustomize.py
-    stage:
-      - -usr/lib/python3.6/sitecustomize.py
-  certbot-apache:
-    plugin: python
-    source: .
-    source-subdir: certbot-apache
-    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
-    python-version: python3
-    after: [python-augeas, certbot]
-    stage-packages: [libaugeas0]
-    stage:
-      # Prefer cffi
-      - -lib/python3.6/site-packages/augeas.py
-  certbot-nginx:
-    plugin: python
-    source: .
-    source-subdir: certbot-nginx
-    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
-    python-version: python3
-    # This is the last step, compile pycache now as there should be no conflicts.
-    override-prime: |
-      snapcraftctl prime
-      ./usr/bin/python3 -m compileall -q .
-    # After certbot-apache to not rebuild duplicates (essentially sharing what was already staged,
-    # like zope)
-    after: [certbot-apache]
+        snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC/certbot && git describe|sed s/^v//`
+
   wrappers:
     plugin: dump
     source: .