Improve github release creation process (#8189)

* Improve github release creation process

* Comment file

* Update tools/create_github_release.py

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* run chmod +x on tools/create_github_release.py

* Add description of create github release method

* remove references to unnecessary azure credential

* remove unnecessary import

* Add reminders to update other file to definitions in .azure-pipelines

* Raise an error if we fail to fetch the artifact from azure

* Create github release as a draft, upload artifacts, then un-draft, for hooks to be run at the right point

* get the version number from the release

* add new packages to dev3_extras so they're installed by tools/venv3.py

* remove unnecessary import

* fun fact: tempdirs behave differently when used as a context manager

* Move comment to construct.py

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -18,6 +18,7 @@ jobs:
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
+          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
           artifact: windows-installer
         displayName: Publish Windows installer
   - job: installer_run

.azure-pipelines/templates/stages/changelog-stage.yml

@@ -5,6 +5,7 @@ stages:
         pool:
           vmImage: vs2017-win2016
         steps:
+          # If we change the output filename from `release_notes.md`, it should also be changed in tools/create_github_release.py
           - bash: |
               CERTBOT_VERSION="$(cd certbot && python -c "import certbot; print(certbot.__version__)" && cd ~-)"
               "${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
@@ -12,5 +13,6 @@ stages:
           - task: PublishPipelineArtifact@1
             inputs:
               path: $(Build.ArtifactStagingDirectory)
+              # If we change the artifact's name, it should also be changed in tools/create_github_release.py
               artifact: changelog
             displayName: Publish changelog

certbot/setup.py

@@ -93,8 +93,10 @@ dev_extras = [
 
 dev3_extras = [
     'astroid',
+    'azure-devops',
     'ipdb',
     'mypy',
+    'PyGithub',
     'pylint',
 ]
 

tools/create_github_release.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+"""
+Post-release script to download artifacts from azure pipelines and use them to create
+a GitHub release.
+
+Setup:
+ - Create a github personal access token
+   - https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token#creating-a-token
+   - You'll need repo scope
+   - Save the token to somewhere like ~/.ssh/githubpat.txt
+
+Run:
+
+python tools/create_github_release.py ~/.ssh/githubpat.txt
+"""
+
+import requests
+import sys
+import tempfile
+from zipfile import ZipFile
+
+from azure.devops.connection import Connection
+from github import Github
+
+def download_azure_artifacts(tempdir):
+    """Download and unzip build artifacts from Azure pipelines.
+
+    :param str path: path to a temporary directory to save the files
+
+    :returns: released certbot version number as a prefix-free string
+    :rtype: str
+
+    """
+    # Create a connection to the azure org
+    organization_url = 'https://dev.azure.com/certbot'
+    connection = Connection(base_url=organization_url)
+
+    # Find the build artifacts
+    build_client = connection.clients.get_build_client()
+    get_builds_response = build_client.get_builds('certbot', definitions='3')
+    build_id = get_builds_response.value[0].id
+    artifacts = build_client.get_artifacts('certbot', build_id)
+
+    # Save and unzip files
+    for filename in ('windows-installer', 'changelog'):
+        url = build_client.get_artifact('certbot', build_id, filename).resource.download_url
+        r = requests.get(url)
+        r.raise_for_status()
+        with open(tempdir + '/' + filename + '.zip', 'wb') as f:
+            f.write(r.content)
+        with ZipFile(tempdir + '/' + filename + '.zip', 'r') as zipObj:
+           zipObj.extractall(tempdir)
+
+    version = build_client.get_build('certbot', build_id).source_branch.split('v')[1]
+    return version
+
+def create_github_release(github_access_token, tempdir, version):
+    """Use build artifacts to create a github release, including uploading additional assets
+
+    :param str github_access_token: string containing github access token
+    :param str path: path to a temporary directory where azure artifacts are located
+    :param str version: Certbot version number, e.g. 1.7.0
+
+    """
+    # Create release
+    g = Github(github_access_token)
+    repo = g.get_user('certbot').get_repo('certbot')
+    release_notes = open(tempdir + '/changelog/release_notes.md', 'r').read()
+    release= repo.create_git_release('v{0}'.format(version),
+                                     'Certbot {0}'.format(version),
+                                     release_notes,
+                                     draft=True)
+
+    # Upload windows installer to release
+    release.upload_asset(tempdir + '/windows-installer/certbot-beta-installer-win32.exe')
+    release.update_release(release.title, release.body, draft=False)
+
+def main(args):
+    github_access_token_file = args[0]
+
+    github_access_token = open(github_access_token_file, 'r').read().rstrip()
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        version = download_azure_artifacts(tempdir)
+        create_github_release(github_access_token, tempdir, version)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

tools/dev_constraints.txt

@@ -9,6 +9,7 @@ appnope==0.1.0
 asn1crypto==0.22.0
 astroid==2.3.3
 attrs==17.3.0
+azure-devops==6.0.0b2
 Babel==2.5.1
 backcall==0.2.0
 backports.functools-lru-cache==1.5
@@ -23,6 +24,7 @@ configparser==3.7.4
 contextlib2==0.6.0.post1
 coverage==4.5.4
 decorator==4.4.1
+deprecated==1.2.10
 dns-lexicon==3.3.17
 dnspython==1.15.0
 docker==3.7.2
@@ -43,6 +45,7 @@ importlib-metadata==0.23
 ipdb==0.12.3
 ipython==7.9.0
 ipython-genutils==0.2.0
+isodate==0.6.0
 isort==4.3.21
 jedi==0.17.1
 Jinja2==2.9.6
@@ -55,10 +58,12 @@ logilab-common==1.4.1
 MarkupSafe==1.0
 mccabe==0.6.1
 more-itertools==5.0.0
+msrest==0.6.18
 mypy==0.710
 mypy-extensions==0.4.3
 ndg-httpsclient==0.3.2
 oauth2client==4.0.0
+oauthlib==3.1.0
 packaging==19.2
 paramiko==2.4.2
 parso==0.7.0
@@ -73,7 +78,9 @@ ptyprocess==0.6.0
 py==1.8.0
 pyasn1==0.1.9
 pyasn1-modules==0.0.10
+PyGithub==1.52
 Pygments==2.2.0
+pyjwt==1.7.1
 pylint==2.4.3
 # If pynsist version is upgraded, our NSIS template windows-installer/template.nsi
 # must be upgraded if necessary using the new built-in one from pynsist.
@@ -91,6 +98,7 @@ pywin32==227
 PyYAML==3.13
 repoze.sphinx.autointerface==0.8
 requests-file==1.4.2
+requests-oauthlib==1.3.0
 requests-toolbelt==0.8.0
 rsa==3.4.2
 s3transfer==0.3.1

windows-installer/construct.py

@@ -123,6 +123,8 @@ imp.load_dynamic('pythoncom', pcom)
     certbot_version = subprocess.check_output([sys.executable, '-c', 'import certbot; print(certbot.__version__)'],
                                               universal_newlines=True, cwd=certbot_pkg_path).strip()
 
+    # If we change the installer name from `certbot-beta-installer-win32.exe`, it should
+    # also be changed in tools/create_github_release.py
     with open(installer_cfg_path, 'w') as file_h:
         file_h.write('''\
 [Application]