diff --git a/scripts/pkg_in_pipe/.gitignore b/scripts/pkg_in_pipe/.gitignore new file mode 100644 index 00000000..e9db76f6 --- /dev/null +++ b/scripts/pkg_in_pipe/.gitignore @@ -0,0 +1,2 @@ +report.html +.cache diff --git a/scripts/pkg_in_pipe/Dockerfile b/scripts/pkg_in_pipe/Dockerfile new file mode 100644 index 00000000..fb62c7f7 --- /dev/null +++ b/scripts/pkg_in_pipe/Dockerfile @@ -0,0 +1,6 @@ +FROM fedora:41 +RUN dnf install -y koji python3-requests python3-pip python3-pygithub +RUN pip install specfile diskcache +VOLUME /tmp/pkg_in_pipe.cache +ADD pkg_in_pipe.py / +ENTRYPOINT ["/pkg_in_pipe.py"] diff --git a/scripts/pkg_in_pipe/README.md b/scripts/pkg_in_pipe/README.md new file mode 100644 index 00000000..3de030d9 --- /dev/null +++ b/scripts/pkg_in_pipe/README.md @@ -0,0 +1,49 @@ +# Packages in pipe report generator + +Generates an html report with the packages in the current tags. + +# Requirements + +You'll need a few extra python modules: +* koji +* requests +* specfile +* pygithub + +The user running the generator must have a working configuration for koji (in `~/.koji`). +A plane token with enough rights to list the cards in the XCPNG project must be passed either through the `PLANE_TOKEN` +environment variable or the `--plane-token` command line option. + +An extra `--generated-info` command line option may be used to add some info about the report generation process. + +# Run in docker + +Before running in docker, the docker image must be built with: + +```sh +docker build -t pkg_in_pipe . +``` + +A volume needs to be available to store the cache: + +```sh +docker volume create pkg_in_pipe_cache +``` + +Several options are required to run the generator in docker: + +* a `PLANE_TOKEN` environment variable with the rights required to request all the cards in the XCPNG project; +* a `GITHUB_TOKEN` environment variable with (at least) the `public_repo` scope; +* a (read only) mount of a directory containing the requeried certificates to connect to koji in `/root/.koji` +* a mount of the output directory in `/output` +* the path of the generated report + +```sh +docker run \ + -v ~/.koji:/root/.koji:z \ + -e PLANE_TOKEN= \ + -e GITHUB_TOKEN= \ + -v /out/dir:/output:z \ + -v pkg_in_pipe_cache:/tmp/pkg_in_pipe.cache \ + pkg_in_pipe /output/index.html +``` diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py new file mode 100755 index 00000000..843f8b87 --- /dev/null +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python + +import argparse +import io +import json +import os +import re +from datetime import datetime +from textwrap import dedent +from typing import cast +from urllib.request import urlopen + +import diskcache +import github +import koji +import requests +from github.Commit import Commit +from github.GithubException import BadCredentialsException +from github.PullRequest import PullRequest + + +def print_header(out): + print(dedent(''' + + + + + + XCP-ng Package Update + + + + '''), file=out) + +def print_plane_warning(out): + print(dedent(''' +
+ +
'''), file=out) + +def print_github_warning(out): + print(dedent(''' +
+ +
'''), file=out) + +def print_koji_error(out): + print(dedent(''' +
+ +
'''), file=out) + +def print_generic_error(out): + print(dedent(''' +
+ +
'''), file=out) + +def print_footer(out, generated_info): + now = datetime.now() + print(dedent(f''' + Last generated at {now}. {generated_info or ''} + + + '''), file=out) + +def print_table_header(out, tag): + print(dedent(f''' +
+
+ + + + + + + + + + + + + '''), file=out) # nopep8 + +def print_table_footer(out): + print(dedent(''' + +
+ {tag} +
+ Build + + Cards + + Pull Requests + + Built by + + Maintained by +
+
+
+ '''), file=out) + +def print_table_line(out, build, link, issues, built_by, prs: list[PullRequest], maintained_by): + issues_content = '\n'.join([ + f'''
  • + XCPNG-{i['sequence_id']} + +
    • ''' + for i in issues + ]) + prs_content = '\n'.join([ + f'''
  • + {pr.title} #{pr.number} + +
    • ''' + for pr in prs + ]) + print(f''' + + + {build} + + + + + + + + + {built_by} + + + {maintained_by if maintained_by is not None else ''} + + + ''', file=out) # nopep8 + +def parse_source(source: str) -> tuple[str, str]: + groups = re.match(r'git\+https://github\.com/([\w-]+/[\w-]+)(|\.git)#([0-9a-f]{40})', source) + assert groups is not None, "can't match the source to the expected github url" + return (groups[1], groups[3]) + +def filter_issues(issues, urls): + res = [] + for issue in issues: + for url in urls: + url = url.strip('/') + if f'href="{url}"' in issue['description_html'] or f'href="{url}/"' in issue['description_html']: + res.append(issue) + break + return res + + +TAG_ORDER = ['incoming', 'ci', 'testing', 'candidates', 'updates', 'base', 'lab'] + +def tag_priority(tag): + # drop the version in the tag — v8.3-incoming -> incoming + tag = tag.split('-')[-1] + return TAG_ORDER.index(tag) + +def find_previous_build_commit(session, build_tag, build): + """Find the previous build in an higher priority koji tag and return its commit.""" + tagged = session.listTagged(build_tag, package=build['package_name'], inherit=True) + tagged = sorted(tagged, key=lambda t: tag_priority(t['tag_name'])) + build_tag_priority = tag_priority(build_tag) + tagged = [t for t in tagged if tag_priority(t['tag_name']) > build_tag_priority] + if not tagged: + return None + previous_build = session.getBuild(tagged[0]['build_id']) + if not previous_build.get('source'): + return None + return parse_source(previous_build['source'])[1] + +def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]: + """ + List the commits in the range [start_sha,end_sha[. + + Note: these are the commits listed by Github starting from start_sha up to end_sha excluded. + A commit older that the end_sha commit and added by a merge commit won't appear in this list. + """ + cache_key = f'commits-{start_sha}-{end_sha}' + if cache_key in CACHE: + return cast(list[Commit], CACHE[cache_key]) + commits = [] + if gh: + for commit in gh.get_repo(repo).get_commits(start_sha): + if commit.sha == end_sha: + break + commits.append(commit) + CACHE[cache_key] = commits + return commits + +def find_pull_requests(gh, repo, start_sha, end_sha): + """Find the pull requests for the commits in the [start_sha,end_sha[ range.""" + prs = set() + for commit in find_commits(gh, repo, start_sha, end_sha): + cache_key = f'commit-prs-{commit.sha}' + if cache_key in CACHE: + prs.update(cast(list[PullRequest], CACHE[cache_key])) + elif gh: + commit_prs = list(commit.get_pulls()) + CACHE[cache_key] = commit_prs + prs.update(commit_prs) + return sorted(prs, key=lambda p: p.number, reverse=True) + +parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') +parser.add_argument('output', nargs='?', help='Report output path', default='report.html') +parser.add_argument('--generated-info', help="Add this message about the generation in the report") +parser.add_argument( + '--plane-token', help="The token used to access the plane api", default=os.environ.get('PLANE_TOKEN') +) +parser.add_argument( + '--github-token', help="The token used to access the Github api", default=os.environ.get('GITHUB_TOKEN') +) +parser.add_argument('--cache', help="The cache path", default="/tmp/pkg_in_pipe.cache") +args = parser.parse_args() + +CACHE = diskcache.Cache(args.cache) + +# load the issues from plane, so we can search for the plane card related to a build +resp = requests.get( + 'https://project.vates.tech/api/v1/workspaces/vates-global/projects/43438eec-1335-4fc2-8804-5a4c32f4932d/issues/', + headers={'x-api-key': args.plane_token}, +) +issues = resp.json().get('results', []) + +# connect to github +if args.github_token: + gh = github.Github(auth=github.Auth.Token(args.github_token)) + try: + gh.get_repo('xcp-ng/xcp') # check that the token is valid + except BadCredentialsException: + gh = None +else: + gh = None + +# load the packages maintainers +with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f: + PACKAGES = json.load(f) + +ok = True +with open(args.output, 'w') as out: + print_header(out) + if not issues: + print_plane_warning(out) + if not gh: + print_github_warning(out) + tags = [f'v{v}-{p}' for v in ['8.2', '8.3'] for p in ['incoming', 'ci', 'testing', 'candidates', 'lab']] + temp_out = io.StringIO() + try: + # open koji session + config = koji.read_config("koji") + session = koji.ClientSession('https://kojihub.xcp-ng.org', config) + session.ssl_login(config['cert'], None, config['serverca']) + for tag in tags: + print_table_header(temp_out, tag) + for tagged in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): + build = session.getBuild(tagged['build_id']) + prs: list[PullRequest] = [] + maintained_by = None + previous_build_sha = find_previous_build_commit(session, tag, build) + if build['source'] is not None: + (repo, sha) = parse_source(build['source']) + prs = find_pull_requests(gh, repo, sha, previous_build_sha) + maintained_by = PACKAGES.get(repo.split('/')[-1], {}).get('maintainer') + build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}' + build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs]) + print_table_line( + temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by + ) + print_table_footer(temp_out) + out.write(temp_out.getvalue()) + except koji.GenericError: + print_koji_error(out) + raise + except Exception: + print_generic_error(out) + raise + finally: + print_footer(out, args.generated_info)