From 20a37036709973eb3614200a9dd604f7b4f8a0e2 Mon Sep 17 00:00:00 2001 From: Sergio Schvezov Date: Thu, 9 Jan 2020 12:41:37 -0300 Subject: [PATCH] Snap plugin * snap: move snapcraft.yaml to snap directory Signed-off-by: Sergio Schvezov * snap: use a local plugin to get around the delivered plugin Add a plugin to the project which behaves as expected until a version of snapcraft satisfies the project needs. Additional snapcraft.yaml changes were made to accommodate for the snap to build. Signed-off-by: Sergio Schvezov * snap: compile pycache in the last step for the last part Signed-off-by: Sergio Schvezov --- snap/snap/plugins/x_python.py | 499 +++++++++++++++++++++++++++++++++ snap/{ => snap}/snapcraft.yaml | 26 +- 2 files changed, 517 insertions(+), 8 deletions(-) create mode 100644 snap/snap/plugins/x_python.py rename snap/{ => snap}/snapcraft.yaml (79%) diff --git a/snap/snap/plugins/x_python.py b/snap/snap/plugins/x_python.py new file mode 100644 index 000000000..5c8c3b86a --- /dev/null +++ b/snap/snap/plugins/x_python.py @@ -0,0 +1,499 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# This is an almost verbatim copy of what is in the snapcraft +# tree, can be completely removed once the functionality in +# snapcraft is in place. (LP: #1841861) +# +# Copyright (C) 2016-2020 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The python plugin can be used for python 2 or 3 based parts. + +It can be used for python projects where you would want to do: + + - import python modules with a requirements.txt + - build a python project that has a setup.py + - install packages straight from pip + +This plugin uses the common plugin keywords as well as those for "sources". +For more information check the 'plugins' topic for the former and the +'sources' topic for the latter. + +Additionally, this plugin uses the following plugin-specific keywords: + + - requirements: + (list of strings) + List of paths to requirements files. + - constraints: + (list of strings) + List of paths to constraint files. + - process-dependency-links: + (bool; default: false) + Enable the processing of dependency links in pip, which allow one + project to provide places to look for another project + - python-packages: + (list) + A list of dependencies to get from PyPI + - python-version: + (string; default: python3) + The python version to use. Valid options are: python2 and python3 + +If the plugin finds a python interpreter with a basename that matches +`python-version` in the directory on the following fixed path: +`/usr/bin/` then this interpreter would +be preferred instead and no interpreter would be brought in through +`stage-packages` mechanisms. +""" + +import collections +import contextlib +import os +import re +from shutil import which +from textwrap import dedent +from typing import List, Optional, Set + +import requests + +import snapcraft +from snapcraft.common import isurl +from snapcraft.internal import errors, mangling +from snapcraft.internal.errors import SnapcraftPluginCommandError +from snapcraft.plugins import _python + + +class UnsupportedPythonVersionError(snapcraft.internal.errors.SnapcraftError): + + fmt = "Unsupported python version: {python_version!r}" + + +class SnapcraftPluginPythonFileMissing(snapcraft.internal.errors.SnapcraftError): + + fmt = ( + "Failed to find the referred {plugin_property} file at the given " + "path: {plugin_property_value!r}.\n" + "Check the property and ensure the file exists." + ) + + def __init__(self, *, plugin_property, plugin_property_value): + super().__init__( + plugin_property=plugin_property, plugin_property_value=plugin_property_value + ) + + +class PythonPlugin(snapcraft.BasePlugin): + @classmethod + def schema(cls): + schema = super().schema() + schema["properties"]["requirements"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["constraints"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["python-packages"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["process-dependency-links"] = { + "type": "boolean", + "default": False, + } + schema["properties"]["python-version"] = { + "type": "string", + "default": "python3", + "enum": ["python2", "python3"], + } + schema["anyOf"] = [{"required": ["source"]}, {"required": ["python-packages"]}] + + return schema + + @classmethod + def get_pull_properties(cls): + # Inform Snapcraft of the properties associated with pulling. If these + # change in the YAML Snapcraft will consider the pull step dirty. + return [ + "requirements", + "constraints", + "python-packages", + "process-dependency-links", + "python-version", + ] + + @property + def plugin_stage_packages(self): + if self.options.python_version == "python2": + python_base = "python" + elif self.options.python_version == "python3": + python_base = "python3" + + if self.project.info.get_build_base() in ("core", "core16", "core18"): + stage_packages = [python_base] + else: + stage_packages = [] + + if self.project.info.get_build_base() == "core18" and python_base == "python3": + stage_packages.append("{}-distutils".format(python_base)) + + return stage_packages + + # ignore mypy error: Read-only property cannot override read-write property + @property # type: ignore + def stage_packages(self): + try: + _python.get_python_command( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ) + except _python.errors.MissingPythonCommandError: + return super().stage_packages + self.plugin_stage_packages + else: + return super().stage_packages + + @property + def _pip(self): + if not self.__pip: + self.__pip = _python.Pip( + python_major_version=self._python_major_version, + part_dir=self.partdir, + install_dir=self.installdir, + stage_dir=self.project.stage_dir, + ) + return self.__pip + + def __init__(self, name, options, project): + super().__init__(name, options, project) + + self._setup_base_tools(project.info.get_build_base()) + + self._manifest = collections.OrderedDict() + + # Pip requires only the major version of python rather than the command + # name like our option requires. + match = re.match(r"python(?P\d).*", self.options.python_version) + if not match: + raise UnsupportedPythonVersionError( + python_version=self.options.python_version + ) + + self._python_major_version = match.group("major_version") + self.__pip = None + + def _setup_base_tools(self, base): + # NOTE: stage-packages are lazily loaded. + if base in ("core", "core16", "core18"): + if self.options.python_version == "python3": + self.build_packages.extend( + [ + "python3-dev", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + ] + ) + elif self.options.python_version == "python2": + self.build_packages.extend( + [ + "python-dev", + "python-pip", + "python-pkg-resources", + "python-setuptools", + ] + ) + else: + raise errors.PluginBaseError(part_name=self.name, base=base) + + def pull(self): + super().pull() + + self._pip.setup() + + with simple_env_bzr(os.path.join(self.installdir, "bin")): + # Download this project, using its setup.py if present. This will + # also download any python-packages requested. + self._download_project() + + def clean_pull(self): + super().clean_pull() + self._pip.clean_packages() + + def build(self): + super().build() + + with simple_env_bzr(os.path.join(self.installdir, "bin")): + # Install the packages that have already been downloaded + installed_pipy_packages = self._install_project() + + requirements = self._get_list_of_packages_from_property( + self.options.requirements + ) + if requirements: + self._manifest["requirements-contents"] = requirements + + constraints = self._get_list_of_packages_from_property(self.options.constraints) + if constraints: + self._manifest["constraints-contents"] = constraints + + self._manifest["python-packages"] = [ + "{}={}".format(name, installed_pipy_packages[name]) + for name in installed_pipy_packages + ] + + try: + _python.generate_sitecustomize( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ) + except _python.errors.MissingUserSitePackagesError as site_error: + print("Part {!r} generated to site-packages: {!s}.".format( + self.name, site_error)) + + def _find_file(self, *, filename: str) -> Optional[str]: + # source-subdir defaults to '' + for basepath in [self.builddir, self.sourcedir]: + if basepath == self.sourcedir: + # This is overwritten in the base plugin + # TODO add consistency + source_subdir = self.options.source_subdir + else: + source_subdir = "" + filepath = os.path.join(basepath, source_subdir, filename) + if os.path.exists(filepath): + return filepath + + return None + + def _get_setup_py_dir(self): + setup_py_dir = None + setup_py_path = self._find_file(filename="setup.py") + if setup_py_path: + setup_py_dir = os.path.dirname(setup_py_path) + + return setup_py_dir + + def _get_list_of_packages_from_property(self, property_list: Set[str]) -> List[str]: + """Return a sorted list of all packages found in property.""" + package_list = list() # type: List[str] + for entry in property_list: + contents = self._get_file_contents(entry) + package_list.extend(contents.splitlines()) + return package_list + + def _get_normalized_property_set( + self, property_name, property_list: List[str] + ) -> Set[str]: + """Return a normalized set from a requirements or constraints list.""" + normalized = set() # type: Set[str] + for entry in property_list: + if isurl(entry): + normalized.add(entry) + else: + entry_file = self._find_file(filename=entry) + if not entry_file: + raise SnapcraftPluginPythonFileMissing( + plugin_property=property_name, plugin_property_value=entry + ) + normalized.add(entry_file) + + return normalized + + def _install_wheels(self, wheels): + installed = self._pip.list() + wheel_names = [os.path.basename(w).split("-")[0] for w in wheels] + + # we want to avoid installing what is already provided in + # stage-packages + need_install = [k for k in wheel_names if k not in installed] + self._pip.install( + need_install, + upgrade=True, + install_deps=False, + process_dependency_links=self.options.process_dependency_links, + ) + + def _download_project(self): + constraints = self._get_normalized_property_set( + "constraints", self.options.constraints + ) + requirements = self._get_normalized_property_set( + "requirements", self.options.requirements + ) + + self._pip.download( + self.options.python_packages, + setup_py_dir=None, + constraints=constraints, + requirements=requirements, + process_dependency_links=self.options.process_dependency_links, + ) + + def _install_project(self): + setup_py_dir = self._get_setup_py_dir() + constraints = self._get_normalized_property_set( + "constraints", self.options.constraints + ) + requirements = self._get_normalized_property_set( + "requirements", self.options.requirements + ) + + # setup.py is handled in a different step as some projects may + # need to satisfy dependencies for setup.py to be parsed. + wheels = self._pip.wheel( + self.options.python_packages, + setup_py_dir=None, + constraints=constraints, + requirements=requirements, + process_dependency_links=self.options.process_dependency_links, + ) + + if wheels: + self._install_wheels(wheels) + + if setup_py_dir is not None: + self._pip.download( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + requirements=set(), + process_dependency_links=self.options.process_dependency_links, + ) + wheels = self._pip.wheel( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + requirements=set(), + process_dependency_links=self.options.process_dependency_links, + ) + + if wheels: + self._install_wheels(wheels) + + setup_py_path = os.path.join(setup_py_dir, "setup.py") + if os.path.exists(setup_py_path): + self._pip.install( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + process_dependency_links=self.options.process_dependency_links, + upgrade=True, + ) + # pbr and others don't work using `pip install .` + # LP: #1670852 + # There is also a chance that this setup.py is distutils based + # in which case we will rely on the `pip install .` ran before + # this. + with contextlib.suppress(SnapcraftPluginCommandError): + self._setup_tools_install(setup_py_path) + + return self._pip.list() + + def _setup_tools_install(self, setup_file): + command = [ + _python.get_python_command( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ), + os.path.basename(setup_file), + "--no-user-cfg", + "install", + "--single-version-externally-managed", + "--user", + "--record", + "install.txt", + ] + self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file)) + + # Fix all shebangs to use the in-snap python. The stuff installed from + # pip has already been fixed, but anything done in this step has not. + mangling.rewrite_python_shebangs(self.installdir) + + def _get_file_contents(self, path): + if isurl(path): + return requests.get(path).text + else: + file_path = os.path.join(self.sourcedir, path) + with open(file_path) as _file: + return _file.read() + + def get_manifest(self): + return self._manifest + + def snap_fileset(self): + fileset = super().snap_fileset() + fileset.append("-bin/pip") + fileset.append("-bin/pip2") + fileset.append("-bin/pip3") + fileset.append("-bin/pip2.7") + fileset.append("-bin/pip3.*") + fileset.append("-bin/easy_install*") + fileset.append("-bin/wheel") + # Holds all the .pyc files. It is a major cause of inter part + # conflict. + fileset.append("-**/__pycache__") + fileset.append("-**/*.pyc") + # The RECORD files include hashes useful when uninstalling packages. + # In the snap they will cause conflicts when more than one part uses + # the python plugin. + fileset.append("-lib/python*/site-packages/*/RECORD") + return fileset + + +@contextlib.contextmanager +def simple_env_bzr(bin_dir): + """Create an appropriate environment to run bzr. + + The python plugin sets up PYTHONUSERBASE and PYTHONHOME which + conflicts with bzr when using python3 as those two environment + variables will make bzr look for modules in the wrong location. + """ + os.makedirs(bin_dir, exist_ok=True) + bzr_bin = os.path.join(bin_dir, "bzr") + real_bzr_bin = which("bzr") + if real_bzr_bin: + exec_line = 'exec {} "$@"'.format(real_bzr_bin) + else: + exec_line = "echo bzr needs to be in PATH; exit 1" + with open(bzr_bin, "w") as f: + f.write( + dedent( + """#!/bin/sh + unset PYTHONUSERBASE + unset PYTHONHOME + {} + """.format( + exec_line + ) + ) + ) + os.chmod(bzr_bin, 0o777) + try: + yield + finally: + os.remove(bzr_bin) + if not os.listdir(bin_dir): + os.rmdir(bin_dir) diff --git a/snap/snapcraft.yaml b/snap/snap/snapcraft.yaml similarity index 79% rename from snap/snapcraft.yaml rename to snap/snap/snapcraft.yaml index 38d32ed0b..0e4a12204 100644 --- a/snap/snapcraft.yaml +++ b/snap/snap/snapcraft.yaml @@ -38,19 +38,19 @@ apps: parts: python-augeas: - plugin: python + plugin: x-python source: git://github.com/basak/python-augeas source-branch: snap python-version: python3 build-packages: [libaugeas-dev] acme: - plugin: python + plugin: x-python source: certbot source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: - plugin: python + plugin: x-python source: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -58,20 +58,30 @@ parts: override-pull: | snapcraftctl pull snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//` + # Workaround for lack of site-packages leading to empty sitecustomize.py + stage: + - -usr/lib/python3.6/sitecustomize.py certbot-apache: - plugin: python + plugin: x-python source: certbot source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [python-augeas, certbot] stage-packages: [libaugeas0] - python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 + stage: + # Prefer cffi + - -lib/python3.6/site-packages/augeas.py certbot-nginx: - plugin: python + plugin: x-python source: certbot source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 - after: [certbot] - python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 + # This is the last step, compile pycache now as there should be no conflicts. + override-prime: | + snapcraftctl prime + ./usr/bin/python3 -m compileall -q . + # After certbot-apache to not rebuild duplicates (essentially sharing what was already staged, + # like zope) + after: [certbot-apache]