llvm-project/llvm/utils/release/github-upload-release.py

#!/usr/bin/env python3
# ===-- github-upload-release.py  ------------------------------------------===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===------------------------------------------------------------------------===#
#
# Create and manage releases in the llvm github project.
#
# This script requires python3 and the PyGithub module.

# Requirements and lockfiles can be found in llvm/utils/git/requirements.upload_release.txt.in
# and llvm/utils/git/requirements.upload_release.txt
#
# Example Usage:
#
# You will need to obtain a personal access token for your github account in
# order to use this script.  Instructions for doing this can be found here:
# https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line
#
# Create a new release from an existing tag:
# ./github-upload-release.py --token $github_token --release 8.0.1-rc4 create
#
# Upload files for a release
# ./github-upload-release.py --token $github_token --release 8.0.1-rc4 upload --files llvm-8.0.1rc4.src.tar.xz
#
# You can upload as many files as you want at a time and use wildcards e.g.
# ./github-upload-release.py --token $github_token --release 8.0.1-rc4 upload --files *.src.*
# ===------------------------------------------------------------------------===#


import argparse
import github
import sys
from textwrap import dedent


# This is a tuple of sections of download links. Each section then contains
# an entry for each line in that section. Each line entry contains:
# * A unique tag to go in "<!-- <tag>" so we can find it later in the release
#   message.
# * A format string for the line's content.
# * A list of filenames to substitute into the format string. Before substitution into
#   the format string, file names will have the base download URL prepended to
#   them and 'release' replaced with the release version.
#
# Between each set of links, an empty line will be added.
#
# This data is used to generate the links for the release message, and by
# uncomment_download_links to verify whether for any given link line, all files
# linked to are present in the release's assets.
release_links = (
    (
        (
            "LINUX_X86",
            "* [Linux x86_64]({0}) ([signature]({1}))",
            (
                "LLVM-{release}-Linux-X64.tar.xz",
                "LLVM-{release}-Linux-X64.tar.xz.jsonl",
            ),
        ),
        (
            "LINUX_ARM64",
            "* [Linux Arm64]({0}) ([signature]({1}))",
            (
                "LLVM-{release}-Linux-ARM64.tar.xz",
                "LLVM-{release}-Linux-ARM64.tar.xz.jsonl",
            ),
        ),
        (
            "LINUX_ARMV7A",
            "* [Linux Armv7-a]({0}) ([signature]({1}))",
            (
                "clang+llvm-{release}-armv7a-linux-gnueabihf.tar.gz",
                "clang+llvm-{release}-armv7a-linux-gnueabihf.tar.gz.sig",
            ),
        ),
    ),
    (
        (
            "MACOS_ARM64",
            "* [macOS Apple Silicon]({0}) (ARM64) ([signature]({1}))",
            (
                "LLVM-{release}-macOS-ARM64.tar.xz",
                "LLVM-{release}-macOS-ARM64.tar.xz.jsonl",
            ),
        ),
        (
            "MACOS_X86",
            "* [macOS Intel]({0}) (x86-64) ([signature]({1}))",
            (
                "LLVM-{release}-macOS-X64.tar.xz",
                "LLVM-{release}-macOS-X64.tar.xz.jsonl",
            ),
        ),
    ),
    (
        (
            "WINDOWS_X64",
            "* Windows x64 (64-bit): [installer]({0}) ([signature]({1})), [archive]({2}) ([signature]({3}))",
            (
                "LLVM-{release}-win64.exe",
                "LLVM-{release}-win64.exe.sig",
                "clang+llvm-{release}-x86_64-pc-windows-msvc.tar.xz",
                "clang+llvm-{release}-x86_64-pc-windows-msvc.tar.xz.sig",
            ),
        ),
        (
            "WINDOWS_X86",
            "* Windows x86 (32-bit): [installer]({0}) ([signature]({1}))",
            ("LLVM-{release}-win32.exe", "LLVM-{release}-win32.exe.sig"),
        ),
        (
            "WINDOWS_ARM64",
            "* Windows on Arm (ARM64): [installer]({0}) ([signature]({1})), [archive]({2}) ([signature]({3}))",
            (
                "LLVM-{release}-woa64.exe",
                "LLVM-{release}-woa64.exe.sig",
                "clang+llvm-{release}-aarch64-pc-windows-msvc.tar.xz",
                "clang+llvm-{release}-aarch64-pc-windows-msvc.tar.xz.sig",
            ),
        ),
    ),
)


def generate_download_links(release):
    base_url = (
        f"https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/"
    )
    markdown_lines = []

    for section in release_links:
        for line in section:
            comment_tag, format_string, files = line
            markdown_line = f"<!-- {comment_tag} "
            files = [base_url + f.format(release=release) for f in files]
            markdown_line += format_string.format(*files)
            markdown_line += " -->"
            markdown_lines.append(markdown_line)

        # Blank line between each section.
        markdown_lines.append("")

    return "\n".join(markdown_lines)


def create_release(repo, release, tag=None, name=None, message=None):
    if not tag:
        tag = "llvmorg-{}".format(release)

    if not name:
        name = "LLVM {}".format(release)

    if not message:
        # Note that these lines are not length limited because if we do so, GitHub
        # assumes that should be how it is laid out on the page. We want GitHub to
        # do the reflowing for us instead.
        download_links = generate_download_links(release)
        message = dedent(
            f"""\
## LLVM {release} Release

{download_links}
Download links for common platforms will appear above once builds have completed, if they are available. Check the full list of release packages at the bottom of this release page if you do not find a link above.

If you do not find a release package for your platform, you may be able to find a community built package on the LLVM Discourse forum thread for this release. Remember that these are built by volunteers and may not always be available. If you rely on a platform or configuration that is not one of the defaults, we suggest you use the binaries that your platform provides, or build your own release packages.

## Package Types

Each platform has one binary release package. The file name starts with either `LLVM-` or `clang+llvm-` and ends with the platform's name. For example, `LLVM-{release}-Linux-ARM64.tar.xz` contains LLVM binaries for Arm64 Linux.

Except for Windows. Where `LLVM-*.exe` is an installer intended for using LLVM as a toolchain and the archive `clang+llvm-` contains the contents of the installer, plus libraries and tools not normally used in a toolchain. You most likely want the `LLVM-` installer, unless you are developing software which itself uses LLVM, in which case choose `clang+llvm-`.

In addition, source archives are available:
* `<sub-project>-{release}.src.tar.xz` are archives of the sources of specific sub-projects of `llvm-project` (except for `test-suite` which is an archive of the [LLVM Test Suite](https://github.com/llvm/llvm-test-suite)).
* To get all the `llvm-project` source code for this release, choose `llvm-project-{release}.src.tar.xz`.

## Verifying Packages

All packages come with a matching `.sig` or `.jsonl` file. You should use these to verify the integrity of the packages.

If it has a `.sig` file, it should have been signed by the release managers using GPG. Download the keys from the [LLVM website](https://releases.llvm.org/release-keys.asc), import them into your keyring and use them to verify the file:
```
$ gpg --import release-keys.asc
$ gpg --verify <package file name>.sig <package file name>
```

If it has a `.jsonl` file, use [gh](https://cli.github.com/manual/gh_attestation_verify) to verify the package:
```
$ gh attestation verify --repo llvm/llvm-project <package file name>
(if you are able to connect to GitHub)
$ gh attestation verify --repo llvm/llvm-project <package file name> --bundle <package file name>.jsonl
(using attestation file on disk)
```"""
        )

    prerelease = True if "rc" in release else False

    repo.create_git_release(tag=tag, name=name, message=message, prerelease=prerelease)


def upload_files(repo, release, files):
    release = repo.get_release("llvmorg-{}".format(release))
    for f in files:
        print("Uploading {}".format(f))
        release.upload_asset(f)
        print("Done")


def uncomment_download_links(repo, release_version):
    release = repo.get_release(f"llvmorg-{release_version}")

    # At this point any automatic builds have finished and if
    # they succeeded, uploaded files to the release assets.
    release_assets = set([a.name for a in release.assets])
    print("Found release assets: ", release_assets)

    new_message = []
    modified = False
    for line in release.body.splitlines():
        # All hidden download links are of the form:
        # <!-- <some unique tag> <markdown content> -->
        if not line.startswith("<!--"):
            new_message.append(line)
            continue

        for section in release_links:
            for comment_tag, _, files in section:
                if not comment_tag in line:
                    continue

                print(f'Found link line "{comment_tag}":')
                files = set([f.format(release=release_version) for f in files])
                print("  Files required:", files)
                if files.issubset(release_assets):
                    print("  All files present, revealing link line.")
                    line = (
                        line.replace("<!--", "")
                        .replace(comment_tag, "")
                        .replace("-->", "")
                        .strip()
                    )
                    modified = True
                else:
                    print(
                        "  These files are not present:",
                        files.difference(release_assets),
                    )
                    print("  Link line will remain hidden.")

        new_message.append(line)

    if modified:
        release.update_release(
            name=release.title,
            message="\n".join(new_message),
            draft=release.draft,
            prerelease=release.prerelease,
        )


parser = argparse.ArgumentParser()
parser.add_argument(
    "command",
    type=str,
    choices=["create", "upload", "check-permissions", "uncomment_download_links"],
)

# All args
parser.add_argument("--token", type=str)
parser.add_argument("--release", type=str)
parser.add_argument("--user", type=str)
parser.add_argument("--user-token", type=str)

# Upload args
parser.add_argument("--files", nargs="+", type=str)

args = parser.parse_args()

gh = github.Github(args.token)
llvm_org = gh.get_organization("llvm")
llvm_repo = llvm_org.get_repo("llvm-project")

if args.user:
    if not args.user_token:
        print("--user-token option required when --user is used")
        sys.exit(1)
    # Validate that this user is allowed to modify releases.
    user = gh.get_user(args.user)
    team = (
        github.Github(args.user_token)
        .get_organization("llvm")
        .get_team_by_slug("llvm-release-managers")
    )
    if not team.has_in_members(user):
        print("User {} is not a allowed to modify releases".format(args.user))
        sys.exit(1)
elif args.command == "check-permissions":
    print("--user option required for check-permissions")
    sys.exit(1)

if args.command == "create":
    create_release(llvm_repo, args.release)
if args.command == "upload":
    upload_files(llvm_repo, args.release, args.files)
if args.command == "uncomment_download_links":
    uncomment_download_links(llvm_repo, args.release)