xcp-ng · glehmann · May 7, 2025 · Feb 28, 2025 · Mar 7, 2025 · Apr 15, 2025
diff --git a/scripts/pkg_in_pipe/.gitignore b/scripts/pkg_in_pipe/.gitignore
@@ -0,0 +1,2 @@
+report.html
+.cache
diff --git a/scripts/pkg_in_pipe/Dockerfile b/scripts/pkg_in_pipe/Dockerfile
@@ -0,0 +1,6 @@
+FROM fedora:41
+RUN dnf install -y koji python3-requests python3-pip python3-pygithub
+RUN pip install specfile diskcache
+VOLUME /tmp/pkg_in_pipe.cache
+ADD pkg_in_pipe.py /
+ENTRYPOINT ["/pkg_in_pipe.py"]
diff --git a/scripts/pkg_in_pipe/README.md b/scripts/pkg_in_pipe/README.md
@@ -0,0 +1,49 @@
+# Packages in pipe report generator
+
+Generates an html report with the packages in the current tags.
+
+# Requirements
+
+You'll need a few extra python modules:
+* koji
+* requests
+* specfile
+* pygithub
+
+The user running the generator must have a working configuration for koji (in `~/.koji`).
+A plane token with enough rights to list the cards in the XCPNG project must be passed either through the `PLANE_TOKEN` 
+environment variable or the `--plane-token` command line option.
+
+An extra `--generated-info` command line option may be used to add some info about the report generation process.
+
+# Run in docker
+
+Before running in docker, the docker image must be built with:
+
+```sh
+docker build -t pkg_in_pipe .
+```
+
+A volume needs to be available to store the cache:
+
+```sh
+docker volume create pkg_in_pipe_cache
+```
+
+Several options are required to run the generator in docker:
+
+* a `PLANE_TOKEN` environment variable with the rights required to request all the cards in the XCPNG project;
+* a `GITHUB_TOKEN` environment variable with (at least) the `public_repo` scope;
+* a (read only) mount of a directory containing the requeried certificates to connect to koji in `/root/.koji`
+* a mount of the output directory in `/output`
+* the path of the generated report
+
+```sh
+docker run \
+    -v ~/.koji:/root/.koji:z \
+    -e PLANE_TOKEN=<plane token> \
+    -e GITHUB_TOKEN=<github token> \
+    -v /out/dir:/output:z \
+    -v pkg_in_pipe_cache:/tmp/pkg_in_pipe.cache \
+    pkg_in_pipe /output/index.html
+```
diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py
@@ -0,0 +1,300 @@
+#!/usr/bin/env python
+
+import argparse
+import io
+import json
+import os
+import re
+from datetime import datetime
+from textwrap import dedent
+from typing import cast
+from urllib.request import urlopen
+
+import diskcache
+import github
+import koji
+import requests
+from github.Commit import Commit
+from github.GithubException import BadCredentialsException
+from github.PullRequest import PullRequest
+
+
+def print_header(out):
+    print(dedent('''
+        <!DOCTYPE html>
+        <html lang="en">
+        <head>
+            <meta charset="UTF-8">
+            <meta name="viewport" content="width=device-width, initial-scale=1.0">
+            <title>XCP-ng Package Update</title>
+            <script src="https://cdn.tailwindcss.com"></script>
+        </head>
+        <body class="bg-gray-400 text-center">
+        '''), file=out)
+
+def print_plane_warning(out):
+    print(dedent('''
+        <div class="px-3 py-3">
+        <div class="bg-orange-100 border-l-4 border-orange-500 text-orange-700 p-4" role="alert">
+            <p class="font-bold">Plane malfunction</p>
+            <p>The issues could not be retrieved from plane.</p>
+        </div>
+        </div>'''), file=out)
+
+def print_github_warning(out):
+    print(dedent('''
+        <div class="px-3 py-3">
+        <div class="bg-orange-100 border-l-4 border-orange-500 text-orange-700 p-4" role="alert">
+            <p class="font-bold">Github access problem</p>
+            <p>The pull requests come from the cache and may not be up to date.</p>
+        </div>
+        </div>'''), file=out)
+
+def print_koji_error(out):
+    print(dedent('''
+        <div class="px-3 py-3">
+        <div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert">
+            <strong class="font-bold">Koji error!</strong>
+            <span class="block sm:inline">The report can't be generated.</span>
+        </div>
+        </div>'''), file=out)
+
+def print_generic_error(out):
+    print(dedent('''
+        <div class="px-3 py-3">
+        <div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert">
+            <strong class="font-bold">Unknown error!</strong>
+            <span class="block sm:inline">The report can't be generated.</span>
+        </div>
+        </div>'''), file=out)
+
+def print_footer(out, generated_info):
+    now = datetime.now()
+    print(dedent(f'''
+        Last generated at {now}. {generated_info or ''}
+        </body>
+        </html>
+        '''), file=out)
+
+def print_table_header(out, tag):
+    print(dedent(f'''
+        <div class="px-3 py-3">
+        <div class="relative overflow-x-auto shadow-md sm:rounded-lg">
+            <table class="table-fixed w-full text-sm text-left rtl:text-right text-gray-500 dark:text-gray-400">
+                <caption class="p-5 text-lg font-semibold text-left rtl:text-right text-gray-900 bg-white dark:text-white dark:bg-gray-800">
+                    {tag}
+                </caption>
+                <thead class="text-xs text-gray-700 uppercase bg-gray-50 dark:bg-gray-700 dark:text-gray-400">
+                    <tr>
+                        <th scope="col" class="px-6 py-3">
+                            Build
+                        </th>
+                        <th scope="col" class="px-6 py-3">
+                            Cards
+                        </th>
+                        <th scope="col" class="px-6 py-3">
+                            Pull Requests
+                        </th>
+                        <th scope="col" class="px-6 py-3">
+                            Built by
+                        </th>
+                        <th scope="col" class="px-6 py-3">
+                            Maintained by
+                        </th>
+                    </tr>
+                </thead>
+                <tbody>
+        '''), file=out)  # nopep8
+
+def print_table_footer(out):
+    print(dedent('''
+                </tbody>
+            </table>
+        </div>
+        </div>
+        '''), file=out)
+
+def print_table_line(out, build, link, issues, built_by, prs: list[PullRequest], maintained_by):
+    issues_content = '\n'.join([
+        f'''<li>
+                <a class="font-medium text-blue-600 dark:text-blue-500 hover:underline"
+                   href="https://project.vates.tech/vates-global/browse/XCPNG-{i['sequence_id']}/">XCPNG-{i['sequence_id']}
+                </a>
+            </li>'''
+        for i in issues
+    ])
+    prs_content = '\n'.join([
+        f'''<li>
+                <a class="font-medium text-blue-600 dark:text-blue-500 hover:underline"
+                   href="{pr.html_url}">{pr.title} #{pr.number}
+                </a>
+            </li>'''
+        for pr in prs
+    ])
+    print(f'''    
+        <tr class="odd:bg-white odd:dark:bg-gray-900 even:bg-gray-50 even:dark:bg-gray-800 border-b dark:border-gray-700 border-gray-200">
+            <th scope="row" class="px-6 py-4 font-medium text-gray-900 whitespace-nowrap dark:text-white">
+                <a class="font-medium text-blue-600 dark:text-blue-500 hover:underline" href="{link}">{build}</a>
+            </th>
+            <td class="px-6 py-4">
+                <ul>
+                {issues_content}
+                </ul>
+            </td>
+            <td class="px-6 py-4">
+                <ul>
+                {prs_content}
+                </ul>
+            </td>
+            <td class="px-6 py-4">
+                {built_by}
+            </td>
+            <td class="px-6 py-4">
+                {maintained_by if maintained_by is not None else ''}
+            </td>
+        </tr>
+        ''', file=out)  # nopep8
+
+def parse_source(source: str) -> tuple[str, str]:
+    groups = re.match(r'git\+https://github\.com/([\w-]+/[\w-]+)(|\.git)#([0-9a-f]{40})', source)
+    assert groups is not None, "can't match the source to the expected github url"
+    return (groups[1], groups[3])
+
+def filter_issues(issues, urls):
+    res = []
+    for issue in issues:
+        for url in urls:
+            url = url.strip('/')
+            if f'href="{url}"' in issue['description_html'] or f'href="{url}/"' in issue['description_html']:
+                res.append(issue)
+                break
+    return res
+
+
+TAG_ORDER = ['incoming', 'ci', 'testing', 'candidates', 'updates', 'base', 'lab']
+
+def tag_priority(tag):
+    # drop the version in the tag — v8.3-incoming -> incoming
+    tag = tag.split('-')[-1]
+    return TAG_ORDER.index(tag)
+
+def find_previous_build_commit(session, build_tag, build):
+    """Find the previous build in an higher priority koji tag and return its commit."""
+    tagged = session.listTagged(build_tag, package=build['package_name'], inherit=True)
+    tagged = sorted(tagged, key=lambda t: tag_priority(t['tag_name']))
+    build_tag_priority = tag_priority(build_tag)
+    tagged = [t for t in tagged if tag_priority(t['tag_name']) > build_tag_priority]
+    if not tagged:
+        return None
+    previous_build = session.getBuild(tagged[0]['build_id'])
+    if not previous_build.get('source'):
+        return None
+    return parse_source(previous_build['source'])[1]
+
+def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]:
+    """
+    List the commits in the range [start_sha,end_sha[.
+
+    Note: these are the commits listed by Github starting from start_sha up to end_sha excluded.
+    A commit older that the end_sha commit and added by a merge commit won't appear in this list.
+    """
+    cache_key = f'commits-{start_sha}-{end_sha}'
+    if cache_key in CACHE:
+        return cast(list[Commit], CACHE[cache_key])
+    commits = []
+    if gh:
+        for commit in gh.get_repo(repo).get_commits(start_sha):
+            if commit.sha == end_sha:
+                break
+            commits.append(commit)
+        CACHE[cache_key] = commits
+    return commits
+
+def find_pull_requests(gh, repo, start_sha, end_sha):
+    """Find the pull requests for the commits in the [start_sha,end_sha[ range."""
+    prs = set()
+    for commit in find_commits(gh, repo, start_sha, end_sha):
+        cache_key = f'commit-prs-{commit.sha}'
+        if cache_key in CACHE:
+            prs.update(cast(list[PullRequest], CACHE[cache_key]))
+        elif gh:
+            commit_prs = list(commit.get_pulls())
+            CACHE[cache_key] = commit_prs
+            prs.update(commit_prs)
+    return sorted(prs, key=lambda p: p.number, reverse=True)
+
+parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe')
+parser.add_argument('output', nargs='?', help='Report output path', default='report.html')
+parser.add_argument('--generated-info', help="Add this message about the generation in the report")
+parser.add_argument(
+    '--plane-token', help="The token used to access the plane api", default=os.environ.get('PLANE_TOKEN')
+)
+parser.add_argument(
+    '--github-token', help="The token used to access the Github api", default=os.environ.get('GITHUB_TOKEN')
+)
+parser.add_argument('--cache', help="The cache path", default="/tmp/pkg_in_pipe.cache")
+args = parser.parse_args()
+
+CACHE = diskcache.Cache(args.cache)
+
+# load the issues from plane, so we can search for the plane card related to a build
+resp = requests.get(
+    'https://project.vates.tech/api/v1/workspaces/vates-global/projects/43438eec-1335-4fc2-8804-5a4c32f4932d/issues/',
+    headers={'x-api-key': args.plane_token},
+)
+issues = resp.json().get('results', [])
+
+# connect to github
+if args.github_token:
+    gh = github.Github(auth=github.Auth.Token(args.github_token))
+    try:
+        gh.get_repo('xcp-ng/xcp')  # check that the token is valid
+    except BadCredentialsException:
+        gh = None
+else:
+    gh = None
+
+# load the packages maintainers
+with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f:
+    PACKAGES = json.load(f)
+
+ok = True
+with open(args.output, 'w') as out:
+    print_header(out)
+    if not issues:
+        print_plane_warning(out)
+    if not gh:
+        print_github_warning(out)
+    tags = [f'v{v}-{p}' for v in ['8.2', '8.3'] for p in ['incoming', 'ci', 'testing', 'candidates', 'lab']]
+    temp_out = io.StringIO()
+    try:
+        # open koji session
+        config = koji.read_config("koji")
+        session = koji.ClientSession('https://kojihub.xcp-ng.org', config)
+        session.ssl_login(config['cert'], None, config['serverca'])
+        for tag in tags:
+            print_table_header(temp_out, tag)
+            for tagged in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True):
+                build = session.getBuild(tagged['build_id'])
+                prs: list[PullRequest] = []
+                maintained_by = None
+                previous_build_sha = find_previous_build_commit(session, tag, build)
+                if build['source'] is not None:
+                    (repo, sha) = parse_source(build['source'])
+                    prs = find_pull_requests(gh, repo, sha, previous_build_sha)
+                    maintained_by = PACKAGES.get(repo.split('/')[-1], {}).get('maintainer')
+                build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}'
+                build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs])
+                print_table_line(
+                    temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by
+                )
+            print_table_footer(temp_out)
+        out.write(temp_out.getvalue())
+    except koji.GenericError:
+        print_koji_error(out)
+        raise
+    except Exception:
+        print_generic_error(out)
+        raise
+    finally:
+        print_footer(out, args.generated_info)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		report.html
nathanael-h marked this conversation as resolved. Show resolved Hide resolved
		.cache