From 622fa7e8ec9c798b19aaa7c3d5223183b17a503a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Fri, 28 Feb 2025 16:00:54 +0100 Subject: [PATCH 01/10] package in pipe html report generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/.gitignore | 1 + scripts/pkg_in_pipe/README.md | 35 +++++++++ scripts/pkg_in_pipe/pkg_in_pipe.py | 120 +++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 scripts/pkg_in_pipe/.gitignore create mode 100644 scripts/pkg_in_pipe/README.md create mode 100755 scripts/pkg_in_pipe/pkg_in_pipe.py diff --git a/scripts/pkg_in_pipe/.gitignore b/scripts/pkg_in_pipe/.gitignore new file mode 100644 index 00000000..6fca42d6 --- /dev/null +++ b/scripts/pkg_in_pipe/.gitignore @@ -0,0 +1 @@ +report.html diff --git a/scripts/pkg_in_pipe/README.md b/scripts/pkg_in_pipe/README.md new file mode 100644 index 00000000..b8b7f3a4 --- /dev/null +++ b/scripts/pkg_in_pipe/README.md @@ -0,0 +1,35 @@ +# Packages in pipe report generator + +Generates an html report with the packages in the current tags. + +# Requirements + +You'll need a few extra python modules: +* koji +* requests +* specfile + +The user running the generator must have a working configuration for koji (in `~/.koji`). +A plane token with enough rights to list the cards in the XCPNG project must be passed either through the `PLANE_TOKEN` +environment variable or the `--plane-token` command line option. + +An extra `--generated-info` command line option may be used to add some info about the report generation process. + + # Run in docker + + Before running in docker, the docker image must be built with: + + ```sh + docker build -t pkg_in_pipe . + ``` + + Several options are required to run the generator in docker: + + * a `PLANE_TOKEN` environment variable with the rights required to request all the cards in the XCPNG project; + * a (read only) mount of a directory containing the requeried certificates to connect to koji in `/root/.koji` + * a mount of the output directory in `/output` + * the path of the generated report + + ```sh + docker run -v ~/.koji:/root/.koji:z -e PLANE_TOKEN= -v /out/dir:/output:z pkg_in_pipe /output/index.html + ``` diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py new file mode 100755 index 00000000..90fe52d9 --- /dev/null +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +import argparse +import os +from datetime import datetime +from textwrap import dedent + +import koji +import requests + + +def print_header(out): + print(dedent(''' + + + + + + XCP-ng Package Update + + + + '''), file=out) + +def print_footer(out, generated_info): + now = datetime.now() + print(dedent(f''' + Last generated at {now}. {generated_info or ''} + + + '''), file=out) + +def print_table_header(out, tag): + print(dedent(f''' +
+
+ + + + + + + + + + + '''), file=out) # nopep8 + +def print_table_footer(out): + print(dedent(''' + +
+ {tag} +
+ Build + + Cards + + By +
+
+
+ '''), file=out) + +def print_table_line(out, build, link, issues, by): + issues_content = '\n'.join([ + f'''
  • + XCPNG-{i['sequence_id']} + +
    • ''' + for i in issues + ]) + print(f''' + + + {build} + + +
      + {issues_content} +
    + + + {by} + + + ''', file=out) # nopep8 + +parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') +parser.add_argument('output', nargs='?', help='Report output path', default='report.html') +parser.add_argument('--generated-info', help="Add this message about the generation in the report") +parser.add_argument( + '--plane-token', help="The token used to access the plane api", default=os.environ.get('PLANE_TOKEN') +) +args = parser.parse_args() + +# open koji session +config = koji.read_config("koji") +session = koji.ClientSession('https://kojihub.xcp-ng.org', config) +session.ssl_login(config['cert'], None, config['serverca']) + +# load the issues from plane, so we can search for the plane card related to a build +resp = requests.get( + 'https://project.vates.tech/api/v1/workspaces/vates-global/projects/43438eec-1335-4fc2-8804-5a4c32f4932d/issues/', + headers={'x-api-key': args.plane_token}, +) +project_issues = resp.json() + +with open(args.output, 'w') as out: + print_header(out) + tags = [f'v{v}-{p}' for v in ['8.2', '8.3'] for p in ['incoming', 'ci', 'testing', 'candidates', 'lab']] + for tag in tags: + print_table_header(out, tag) + for build in session.listTagged(tag): + build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={build['build_id']}' + build_issues = [i for i in project_issues['results'] if f'href="{build_url}"' in i['description_html']] + print_table_line(out, build['nvr'], build_url, build_issues, build['owner_name']) + print_table_footer(out) + print_footer(out, args.generated_info) From 74c05e8231556b7cd6037d0c3521c24ca94c09f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Fri, 7 Mar 2025 10:52:37 +0100 Subject: [PATCH 02/10] run pkg_in_pipe in docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/pkg_in_pipe/Dockerfile diff --git a/scripts/pkg_in_pipe/Dockerfile b/scripts/pkg_in_pipe/Dockerfile new file mode 100644 index 00000000..eb9b6317 --- /dev/null +++ b/scripts/pkg_in_pipe/Dockerfile @@ -0,0 +1,5 @@ +FROM fedora:41 +RUN dnf install -y koji python3-requests python3-pip +RUN pip install specfile +ADD pkg_in_pipe.py / +ENTRYPOINT ["/pkg_in_pipe.py"] From bf0346cc7465239022231596278cda004f7cf5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Tue, 15 Apr 2025 10:56:29 +0200 Subject: [PATCH 03/10] don't fail when plane or koji are not responding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/pkg_in_pipe.py | 68 ++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index 90fe52d9..cfb82dd6 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import argparse +import io import os from datetime import datetime from textwrap import dedent @@ -22,6 +23,33 @@ def print_header(out): '''), file=out) +def print_plane_warning(out): + print(dedent(''' +
    + +
    '''), file=out) + +def print_koji_error(out): + print(dedent(''' +
    + +
    '''), file=out) + +def print_generic_error(out): + print(dedent(''' +
    + +
    '''), file=out) + def print_footer(out, generated_info): now = datetime.now() print(dedent(f''' @@ -95,26 +123,38 @@ def print_table_line(out, build, link, issues, by): ) args = parser.parse_args() -# open koji session -config = koji.read_config("koji") -session = koji.ClientSession('https://kojihub.xcp-ng.org', config) -session.ssl_login(config['cert'], None, config['serverca']) - # load the issues from plane, so we can search for the plane card related to a build resp = requests.get( 'https://project.vates.tech/api/v1/workspaces/vates-global/projects/43438eec-1335-4fc2-8804-5a4c32f4932d/issues/', headers={'x-api-key': args.plane_token}, ) -project_issues = resp.json() +issues = resp.json().get('results', []) +ok = True with open(args.output, 'w') as out: print_header(out) + if not issues: + print_plane_warning(out) tags = [f'v{v}-{p}' for v in ['8.2', '8.3'] for p in ['incoming', 'ci', 'testing', 'candidates', 'lab']] - for tag in tags: - print_table_header(out, tag) - for build in session.listTagged(tag): - build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={build['build_id']}' - build_issues = [i for i in project_issues['results'] if f'href="{build_url}"' in i['description_html']] - print_table_line(out, build['nvr'], build_url, build_issues, build['owner_name']) - print_table_footer(out) - print_footer(out, args.generated_info) + temp_out = io.StringIO() + try: + # open koji session + config = koji.read_config("koji") + session = koji.ClientSession('https://kojihub.xcp-ng.org', config) + session.ssl_login(config['cert'], None, config['serverca']) + for tag in tags: + print_table_header(temp_out, tag) + for build in session.listTagged(tag): + build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={build['build_id']}' + build_issues = [i for i in issues if f'href="{build_url}"' in i['description_html']] + print_table_line(temp_out, build['nvr'], build_url, build_issues, build['owner_name']) + print_table_footer(temp_out) + out.write(temp_out.getvalue()) + except koji.GenericError: + print_koji_error(out) + raise + except Exception: + print_generic_error(out) + raise + finally: + print_footer(out, args.generated_info) From dc7daa3daf1c3ae853f21a177342e342836995e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Tue, 15 Apr 2025 13:53:42 +0200 Subject: [PATCH 04/10] sort the packages by reversed build id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/pkg_in_pipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index cfb82dd6..84e09ea1 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -144,7 +144,7 @@ def print_table_line(out, build, link, issues, by): session.ssl_login(config['cert'], None, config['serverca']) for tag in tags: print_table_header(temp_out, tag) - for build in session.listTagged(tag): + for build in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={build['build_id']}' build_issues = [i for i in issues if f'href="{build_url}"' in i['description_html']] print_table_line(temp_out, build['nvr'], build_url, build_issues, build['owner_name']) From 725ad1d82d70364138e4a444308a3b245f5dc2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Fri, 25 Apr 2025 10:51:58 +0200 Subject: [PATCH 05/10] add the pull request matching the koji build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann Co-authored-by: Nathanaël <7300309+nathanael-h@users.noreply.github.com> --- scripts/pkg_in_pipe/Dockerfile | 2 +- scripts/pkg_in_pipe/README.md | 1 + scripts/pkg_in_pipe/pkg_in_pipe.py | 40 +++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/scripts/pkg_in_pipe/Dockerfile b/scripts/pkg_in_pipe/Dockerfile index eb9b6317..12bfe0a8 100644 --- a/scripts/pkg_in_pipe/Dockerfile +++ b/scripts/pkg_in_pipe/Dockerfile @@ -1,5 +1,5 @@ FROM fedora:41 -RUN dnf install -y koji python3-requests python3-pip +RUN dnf install -y koji python3-requests python3-pip python3-pygithub RUN pip install specfile ADD pkg_in_pipe.py / ENTRYPOINT ["/pkg_in_pipe.py"] diff --git a/scripts/pkg_in_pipe/README.md b/scripts/pkg_in_pipe/README.md index b8b7f3a4..1b87696f 100644 --- a/scripts/pkg_in_pipe/README.md +++ b/scripts/pkg_in_pipe/README.md @@ -26,6 +26,7 @@ An extra `--generated-info` command line option may be used to add some info abo Several options are required to run the generator in docker: * a `PLANE_TOKEN` environment variable with the rights required to request all the cards in the XCPNG project; + * a `GITHUB_TOKEN` environment variable with (at least) the `public_repo` scope; * a (read only) mount of a directory containing the requeried certificates to connect to koji in `/root/.koji` * a mount of the output directory in `/output` * the path of the generated report diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index 84e09ea1..972ff4b4 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -3,11 +3,14 @@ import argparse import io import os +import re from datetime import datetime from textwrap import dedent +import github import koji import requests +from github.PullRequest import PullRequest def print_header(out): @@ -74,6 +77,9 @@ def print_table_header(out, tag): Cards + + Pull Requests + By @@ -90,7 +96,7 @@ def print_table_footer(out): '''), file=out) -def print_table_line(out, build, link, issues, by): +def print_table_line(out, build, link, issues, by, prs: list[PullRequest]): issues_content = '\n'.join([ f'''
  • ''' for i in issues ]) + prs_content = '\n'.join([ + f'''
  • + {pr.title} #{pr.number} + +
    • ''' + for pr in prs + ]) print(f''' @@ -109,12 +123,22 @@ def print_table_line(out, build, link, issues, by): {issues_content} + +
      + {prs_content} +
    + {by} ''', file=out) # nopep8 +def parse_source(source: str) -> tuple[str, str]: + groups = re.match(r'git\+https://github\.com/([\w-]+/[\w-]+)(|\.git)#([0-9a-f]{40})', source) + assert groups is not None, "can't match the source to the expected github url" + return (groups[1], groups[3]) + parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') parser.add_argument('output', nargs='?', help='Report output path', default='report.html') parser.add_argument('--generated-info', help="Add this message about the generation in the report") @@ -130,6 +154,9 @@ def print_table_line(out, build, link, issues, by): ) issues = resp.json().get('results', []) +# connect to github +gh = github.Github(auth=github.Auth.Token(os.environ['GITHUB_TOKEN'])) + ok = True with open(args.output, 'w') as out: print_header(out) @@ -144,10 +171,15 @@ def print_table_line(out, build, link, issues, by): session.ssl_login(config['cert'], None, config['serverca']) for tag in tags: print_table_header(temp_out, tag) - for build in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): - build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={build['build_id']}' + for tagged in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): + build = session.getBuild(tagged['build_id']) + prs: list[PullRequest] = [] + if build['source'] is not None: + (repo, sha) = parse_source(build['source']) + prs = list(gh.get_repo(repo).get_commit(sha).get_pulls()) + build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}' build_issues = [i for i in issues if f'href="{build_url}"' in i['description_html']] - print_table_line(temp_out, build['nvr'], build_url, build_issues, build['owner_name']) + print_table_line(temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs) print_table_footer(temp_out) out.write(temp_out.getvalue()) except koji.GenericError: From 37f8c7fd11326899d2ea87f3778282268cd27193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Fri, 25 Apr 2025 16:07:18 +0200 Subject: [PATCH 06/10] also search the plane cards with the PR urls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/pkg_in_pipe.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index 972ff4b4..e5b86535 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -139,6 +139,17 @@ def parse_source(source: str) -> tuple[str, str]: assert groups is not None, "can't match the source to the expected github url" return (groups[1], groups[3]) +def filter_issues(issues, urls): + res = [] + for issue in issues: + for url in urls: + url = url.strip('/') + if f'href="{url}"' in issue['description_html'] or f'href="{url}/"' in issue['description_html']: + res.append(issue) + break + return res + + parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') parser.add_argument('output', nargs='?', help='Report output path', default='report.html') parser.add_argument('--generated-info', help="Add this message about the generation in the report") @@ -178,7 +189,7 @@ def parse_source(source: str) -> tuple[str, str]: (repo, sha) = parse_source(build['source']) prs = list(gh.get_repo(repo).get_commit(sha).get_pulls()) build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}' - build_issues = [i for i in issues if f'href="{build_url}"' in i['description_html']] + build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs]) print_table_line(temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs) print_table_footer(temp_out) out.write(temp_out.getvalue()) From 7fbe0701e6c5eb704a6e5df7e37e8cb901132075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Mon, 28 Apr 2025 11:08:56 +0200 Subject: [PATCH 07/10] include all the PRs merged after the previous build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/pkg_in_pipe.py | 33 +++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index e5b86535..1be46be8 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -150,6 +150,36 @@ def filter_issues(issues, urls): return res +TAG_ORDER = ['incoming', 'ci', 'testing', 'candidates', 'updates', 'base', 'lab'] + +def tag_priority(tag): + # drop the version in the tag — v8.3-incoming -> incoming + tag = tag.split('-')[-1] + return TAG_ORDER.index(tag) + +def find_previous_build_commit(session, build_tag, build): + """Find the previous build in an higher priority koji tag and return its commit.""" + tagged = session.listTagged(build_tag, package=build['package_name'], inherit=True) + tagged = sorted(tagged, key=lambda t: tag_priority(t['tag_name'])) + build_tag_priority = tag_priority(build_tag) + tagged = [t for t in tagged if tag_priority(t['tag_name']) > build_tag_priority] + if not tagged: + return None + previous_build = session.getBuild(tagged[0]['build_id']) + if not previous_build.get('source'): + return None + return parse_source(previous_build['source'])[1] + +def find_pull_requests(gh, repo, start_sha, end_sha): + """Find the pull requests for the commits in the [start_sha,end_sha[ range.""" + prs = set() + for commit in gh.get_repo(repo).get_commits(start_sha): + if commit.sha == end_sha: + break + for pr in commit.get_pulls(): + prs.add(pr) + return sorted(prs, key=lambda p: p.number, reverse=True) + parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') parser.add_argument('output', nargs='?', help='Report output path', default='report.html') parser.add_argument('--generated-info', help="Add this message about the generation in the report") @@ -185,9 +215,10 @@ def filter_issues(issues, urls): for tagged in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): build = session.getBuild(tagged['build_id']) prs: list[PullRequest] = [] + previous_build_sha = find_previous_build_commit(session, tag, build) if build['source'] is not None: (repo, sha) = parse_source(build['source']) - prs = list(gh.get_repo(repo).get_commit(sha).get_pulls()) + prs = find_pull_requests(gh, repo, sha, previous_build_sha) build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}' build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs]) print_table_line(temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs) From ec8652fd0ba768072f7c1f6c4959bea411c72bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Mon, 28 Apr 2025 12:09:08 +0200 Subject: [PATCH 08/10] cache github calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/.gitignore | 1 + scripts/pkg_in_pipe/Dockerfile | 3 +- scripts/pkg_in_pipe/README.md | 51 +++++++++++++++++++----------- scripts/pkg_in_pipe/pkg_in_pipe.py | 37 +++++++++++++++++++--- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/scripts/pkg_in_pipe/.gitignore b/scripts/pkg_in_pipe/.gitignore index 6fca42d6..e9db76f6 100644 --- a/scripts/pkg_in_pipe/.gitignore +++ b/scripts/pkg_in_pipe/.gitignore @@ -1 +1,2 @@ report.html +.cache diff --git a/scripts/pkg_in_pipe/Dockerfile b/scripts/pkg_in_pipe/Dockerfile index 12bfe0a8..fb62c7f7 100644 --- a/scripts/pkg_in_pipe/Dockerfile +++ b/scripts/pkg_in_pipe/Dockerfile @@ -1,5 +1,6 @@ FROM fedora:41 RUN dnf install -y koji python3-requests python3-pip python3-pygithub -RUN pip install specfile +RUN pip install specfile diskcache +VOLUME /tmp/pkg_in_pipe.cache ADD pkg_in_pipe.py / ENTRYPOINT ["/pkg_in_pipe.py"] diff --git a/scripts/pkg_in_pipe/README.md b/scripts/pkg_in_pipe/README.md index 1b87696f..3de030d9 100644 --- a/scripts/pkg_in_pipe/README.md +++ b/scripts/pkg_in_pipe/README.md @@ -8,6 +8,7 @@ You'll need a few extra python modules: * koji * requests * specfile +* pygithub The user running the generator must have a working configuration for koji (in `~/.koji`). A plane token with enough rights to list the cards in the XCPNG project must be passed either through the `PLANE_TOKEN` @@ -15,22 +16,34 @@ environment variable or the `--plane-token` command line option. An extra `--generated-info` command line option may be used to add some info about the report generation process. - # Run in docker - - Before running in docker, the docker image must be built with: - - ```sh - docker build -t pkg_in_pipe . - ``` - - Several options are required to run the generator in docker: - - * a `PLANE_TOKEN` environment variable with the rights required to request all the cards in the XCPNG project; - * a `GITHUB_TOKEN` environment variable with (at least) the `public_repo` scope; - * a (read only) mount of a directory containing the requeried certificates to connect to koji in `/root/.koji` - * a mount of the output directory in `/output` - * the path of the generated report - - ```sh - docker run -v ~/.koji:/root/.koji:z -e PLANE_TOKEN= -v /out/dir:/output:z pkg_in_pipe /output/index.html - ``` +# Run in docker + +Before running in docker, the docker image must be built with: + +```sh +docker build -t pkg_in_pipe . +``` + +A volume needs to be available to store the cache: + +```sh +docker volume create pkg_in_pipe_cache +``` + +Several options are required to run the generator in docker: + +* a `PLANE_TOKEN` environment variable with the rights required to request all the cards in the XCPNG project; +* a `GITHUB_TOKEN` environment variable with (at least) the `public_repo` scope; +* a (read only) mount of a directory containing the requeried certificates to connect to koji in `/root/.koji` +* a mount of the output directory in `/output` +* the path of the generated report + +```sh +docker run \ + -v ~/.koji:/root/.koji:z \ + -e PLANE_TOKEN= \ + -e GITHUB_TOKEN= \ + -v /out/dir:/output:z \ + -v pkg_in_pipe_cache:/tmp/pkg_in_pipe.cache \ + pkg_in_pipe /output/index.html +``` diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index 1be46be8..f6889773 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -6,10 +6,13 @@ import re from datetime import datetime from textwrap import dedent +from typing import cast +import diskcache import github import koji import requests +from github.Commit import Commit from github.PullRequest import PullRequest @@ -170,14 +173,35 @@ def find_previous_build_commit(session, build_tag, build): return None return parse_source(previous_build['source'])[1] -def find_pull_requests(gh, repo, start_sha, end_sha): - """Find the pull requests for the commits in the [start_sha,end_sha[ range.""" - prs = set() +def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]: + """ + List the commits in the range [start_sha,end_sha[. + + Note: these are the commits listed by Github starting from start_sha up to end_sha excluded. + A commit older that the end_sha commit and added by a merge commit won't appear in this list. + """ + cache_key = f'commits-{start_sha}-{end_sha}' + if cache_key in CACHE: + return cast(list[Commit], CACHE[cache_key]) + commits = [] for commit in gh.get_repo(repo).get_commits(start_sha): if commit.sha == end_sha: break - for pr in commit.get_pulls(): - prs.add(pr) + commits.append(commit) + CACHE[cache_key] = commits + return commits + +def find_pull_requests(gh, repo, start_sha, end_sha): + """Find the pull requests for the commits in the [start_sha,end_sha[ range.""" + prs = set() + for commit in find_commits(gh, repo, start_sha, end_sha): + cache_key = f'commit-prs-{commit.sha}' + if cache_key in CACHE: + prs.update(cast(list[PullRequest], CACHE[cache_key])) + else: + commit_prs = list(commit.get_pulls()) + CACHE[cache_key] = commit_prs + prs.update(commit_prs) return sorted(prs, key=lambda p: p.number, reverse=True) parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') @@ -186,8 +210,11 @@ def find_pull_requests(gh, repo, start_sha, end_sha): parser.add_argument( '--plane-token', help="The token used to access the plane api", default=os.environ.get('PLANE_TOKEN') ) +parser.add_argument('--cache', help="The cache path", default="/tmp/pkg_in_pipe.cache") args = parser.parse_args() +CACHE = diskcache.Cache(args.cache) + # load the issues from plane, so we can search for the plane card related to a build resp = requests.get( 'https://project.vates.tech/api/v1/workspaces/vates-global/projects/43438eec-1335-4fc2-8804-5a4c32f4932d/issues/', From 0a5bd0c98466d78349abfa2466d21def3a810745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Tue, 29 Apr 2025 09:42:50 +0200 Subject: [PATCH 09/10] generate the report even without github MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/pkg_in_pipe.py | 37 ++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index f6889773..dcda4dec 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -13,6 +13,7 @@ import koji import requests from github.Commit import Commit +from github.GithubException import BadCredentialsException from github.PullRequest import PullRequest @@ -38,6 +39,15 @@ def print_plane_warning(out): '''), file=out) +def print_github_warning(out): + print(dedent(''' +
    + +
    '''), file=out) + def print_koji_error(out): print(dedent('''
    @@ -184,11 +194,12 @@ def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]: if cache_key in CACHE: return cast(list[Commit], CACHE[cache_key]) commits = [] - for commit in gh.get_repo(repo).get_commits(start_sha): - if commit.sha == end_sha: - break - commits.append(commit) - CACHE[cache_key] = commits + if gh: + for commit in gh.get_repo(repo).get_commits(start_sha): + if commit.sha == end_sha: + break + commits.append(commit) + CACHE[cache_key] = commits return commits def find_pull_requests(gh, repo, start_sha, end_sha): @@ -198,7 +209,7 @@ def find_pull_requests(gh, repo, start_sha, end_sha): cache_key = f'commit-prs-{commit.sha}' if cache_key in CACHE: prs.update(cast(list[PullRequest], CACHE[cache_key])) - else: + elif gh: commit_prs = list(commit.get_pulls()) CACHE[cache_key] = commit_prs prs.update(commit_prs) @@ -210,6 +221,9 @@ def find_pull_requests(gh, repo, start_sha, end_sha): parser.add_argument( '--plane-token', help="The token used to access the plane api", default=os.environ.get('PLANE_TOKEN') ) +parser.add_argument( + '--github-token', help="The token used to access the Github api", default=os.environ.get('GITHUB_TOKEN') +) parser.add_argument('--cache', help="The cache path", default="/tmp/pkg_in_pipe.cache") args = parser.parse_args() @@ -223,13 +237,22 @@ def find_pull_requests(gh, repo, start_sha, end_sha): issues = resp.json().get('results', []) # connect to github -gh = github.Github(auth=github.Auth.Token(os.environ['GITHUB_TOKEN'])) +if args.github_token: + gh = github.Github(auth=github.Auth.Token(args.github_token)) + try: + gh.get_repo('xcp-ng/xcp') # check that the token is valid + except BadCredentialsException: + gh = None +else: + gh = None ok = True with open(args.output, 'w') as out: print_header(out) if not issues: print_plane_warning(out) + if not gh: + print_github_warning(out) tags = [f'v{v}-{p}' for v in ['8.2', '8.3'] for p in ['incoming', 'ci', 'testing', 'candidates', 'lab']] temp_out = io.StringIO() try: From 31f4928420304e921e65aa8626a037bf0bae19ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Tue, 29 Apr 2025 16:23:55 +0200 Subject: [PATCH 10/10] add a maintainer column MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gaëtan Lehmann --- scripts/pkg_in_pipe/pkg_in_pipe.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/pkg_in_pipe/pkg_in_pipe.py b/scripts/pkg_in_pipe/pkg_in_pipe.py index dcda4dec..843f8b87 100755 --- a/scripts/pkg_in_pipe/pkg_in_pipe.py +++ b/scripts/pkg_in_pipe/pkg_in_pipe.py @@ -2,11 +2,13 @@ import argparse import io +import json import os import re from datetime import datetime from textwrap import dedent from typing import cast +from urllib.request import urlopen import diskcache import github @@ -94,7 +96,10 @@ def print_table_header(out, tag): Pull Requests - By + Built by + + + Maintained by @@ -109,7 +114,7 @@ def print_table_footer(out):
    '''), file=out) -def print_table_line(out, build, link, issues, by, prs: list[PullRequest]): +def print_table_line(out, build, link, issues, built_by, prs: list[PullRequest], maintained_by): issues_content = '\n'.join([ f'''
  • - {by} + {built_by} + + + {maintained_by if maintained_by is not None else ''} ''', file=out) # nopep8 @@ -246,6 +254,10 @@ def find_pull_requests(gh, repo, start_sha, end_sha): else: gh = None +# load the packages maintainers +with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f: + PACKAGES = json.load(f) + ok = True with open(args.output, 'w') as out: print_header(out) @@ -265,13 +277,17 @@ def find_pull_requests(gh, repo, start_sha, end_sha): for tagged in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): build = session.getBuild(tagged['build_id']) prs: list[PullRequest] = [] + maintained_by = None previous_build_sha = find_previous_build_commit(session, tag, build) if build['source'] is not None: (repo, sha) = parse_source(build['source']) prs = find_pull_requests(gh, repo, sha, previous_build_sha) + maintained_by = PACKAGES.get(repo.split('/')[-1], {}).get('maintainer') build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}' build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs]) - print_table_line(temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs) + print_table_line( + temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by + ) print_table_footer(temp_out) out.write(temp_out.getvalue()) except koji.GenericError: