diff --git a/semantic/generators/GenerateMavenVersions.java b/semantic/generators/GenerateMavenVersions.java
new file mode 100755
index 00000000..f0816852
--- /dev/null
+++ b/semantic/generators/GenerateMavenVersions.java
@@ -0,0 +1,291 @@
+import org.apache.maven.artifact.versioning.ComparableVersion;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.io.*;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+/**
+ * Script for generating a list of maven version comparison fixtures based off
+ * every version mentioned in the OSV Maven database, sorted using the native
+ * Maven implementation.
+ *
+ * To run this, you need to ensure copies of the following libraries are present
+ * on the class path:
+ *
+ *
+ * The easiest way to do this is by putting the jars into a lib
subfolder and then running:
+ *
+ * java -cp generators/lib/* generators/GenerateMavenVersions.java
+ *
+ */
+public class GenerateMavenVersions {
+ /**
+ * An array of version comparisons that are known to be unsupported and so
+ * should be commented out in the generated fixture.
+ *
+ * Generally this is because the native implementation has a suspected bug
+ * that causes the comparison to return incorrect results, and so supporting
+ * such comparisons in the detector would in fact be wrong.
+ */
+ private static final String[] UNSUPPORTED_COMPARISONS = {
+ "0.0.0-2021-07-06T00-28-13-573087f7 < 0.0.0-2021-07-06T01-14-42-efe42242",
+ "0.0.0-2021-12-06T00-08-57-89a33731 < 0.0.0-2021-12-06T01-21-56-e3888760",
+ "0.0.0-2022-02-01T00-45-53-0300684a < 0.0.0-2022-02-01T05-45-16-7258ece0",
+ "0.0.0-2022-02-28T00-18-39-7fe0d845 < 0.0.0-2022-02-28T04-15-47-83c97ebe",
+ "0.0.0-2022-04-29T00-08-11-7086a3ec < 0.0.0-2022-04-29T01-20-09-b424f986",
+ "0.0.0-2022-06-14T00-21-33-f21869a7 < 0.0.0-2022-06-14T02-56-29-1db980e0",
+ "0.0.0-2022-08-16T00-14-19-aeae3dc3 < 0.0.0-2022-08-16T10-34-26-7a56f709",
+ "0.0.0-2022-08-22T00-46-32-4652d3db < 0.0.0-2022-08-22T06-46-40-e7409ac5",
+ "0.0.0-2022-10-31T00-42-12-322ba6b9 < 0.0.0-2022-10-31T01-23-06-c6652489",
+ "0.0.0-2022-10-31T07-00-43-71eccd49 < 0.0.0-2022-10-31T07-05-43-97874976",
+ "0.0.0-2022-12-01T00-02-29-fe8d6705 < 0.0.0-2022-12-01T01-56-22-5b442198",
+ "0.0.0-2022-12-18T00-44-34-a222f475 < 0.0.0-2022-12-18T01-45-19-fec81751",
+ "0.0.0-2023-03-20T00-52-15-4b4c0e7 < 0.0.0-2023-03-20T01-49-44-80e3135"
+ };
+
+ public static boolean isUnsupportedComparison(String line) {
+ return Arrays.stream(UNSUPPORTED_COMPARISONS).anyMatch(line::equals);
+ }
+
+ public static String uncomment(String line) {
+ if(line.startsWith("#")) {
+ return line.substring(1);
+ }
+
+ if(line.startsWith("//")) {
+ return line.substring(2);
+ }
+
+ return line;
+ }
+
+ public static String downloadMavenDb() throws IOException {
+ URL website = new URL("https://osv-vulnerabilities.storage.googleapis.com/Maven/all.zip");
+ String file = "./maven-db.zip";
+
+ ReadableByteChannel rbc = Channels.newChannel(website.openStream());
+
+ try(FileOutputStream fos = new FileOutputStream(file)) {
+ fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+ }
+
+ return file;
+ }
+
+ public static Map> fetchPackageVersions() throws IOException {
+ String dbPath = downloadMavenDb();
+ List osvs = loadOSVs(dbPath);
+
+ Map> packages = new HashMap<>();
+
+ osvs.forEach(osv -> osv.getJSONArray("affected").forEach(aff -> {
+ JSONObject affected = (JSONObject) aff;
+
+ if(!affected.has("package") || affected.getJSONObject("package").getString("ecosystem").equals("Maven")) {
+ return;
+ }
+
+ String pkgName = affected.getJSONObject("package").getString("name");
+
+ if(!affected.has("versions")) {
+ return;
+ }
+ JSONArray versions = affected.getJSONArray("versions");
+
+ packages.putIfAbsent(pkgName, new ArrayList<>());
+
+ if(versions.isEmpty()) {
+ return;
+ }
+
+ versions.forEach(version -> packages.get(pkgName).add((String) version));
+ }));
+
+ packages.forEach((key, _ignore) -> packages.put(
+ key,
+ packages.get(key)
+ .stream()
+ .distinct()
+ .sorted(Comparator.comparing(ComparableVersion::new))
+ .collect(Collectors.toList())
+ ));
+
+ return packages;
+ }
+
+ public static List loadOSVs(String pathToDbZip) throws IOException {
+ List osvs = new ArrayList<>();
+
+ try(ZipFile zipFile = new ZipFile(pathToDbZip)) {
+ Enumeration extends ZipEntry> entries = zipFile.entries();
+
+ while(entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ InputStream stream = zipFile.getInputStream(entry);
+
+ BufferedReader streamReader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
+ StringBuilder responseStrBuilder = new StringBuilder();
+
+ String inputStr;
+ while((inputStr = streamReader.readLine()) != null) {
+ responseStrBuilder.append(inputStr);
+ }
+ osvs.add(new JSONObject(responseStrBuilder.toString()));
+ }
+ }
+
+ return osvs;
+ }
+
+ public static void writeToFile(String outfile, List lines) throws IOException {
+ try(PrintWriter writer = new PrintWriter(outfile, StandardCharsets.UTF_8)) {
+ lines.forEach(writer::println);
+ }
+ }
+
+ public static boolean compareVers(String version1, String op, String version2) {
+ ComparableVersion v1 = new ComparableVersion(version1);
+ ComparableVersion v2 = new ComparableVersion(version2);
+
+ int r = v1.compareTo(v2);
+
+ if(op.equals("=")) {
+ return r == 0;
+ }
+
+ if(op.equals("<")) {
+ return r < 0;
+ }
+
+ if(op.equals(">")) {
+ return r > 0;
+ }
+
+ throw new RuntimeException("unsupported comparison operator " + op);
+ }
+
+ public static boolean compareVersions(List lines, String select) {
+ boolean didAnyFail = false;
+
+ for(String line : lines) {
+ line = line.trim();
+
+ if(line.isEmpty() || line.startsWith("#") || line.startsWith("//")) {
+ String maybeUnsupported = uncomment(line).trim();
+
+ if(isUnsupportedComparison(maybeUnsupported)) {
+ System.out.printf("\033[96mS\033[0m: \033[93m%s\033[0m\n", maybeUnsupported);
+ }
+
+ continue;
+ }
+
+ String[] parts = line.split(" ");
+ String v1 = parts[0];
+ String op = parts[1];
+ String v2 = parts[2];
+
+ boolean r = compareVers(v1, op, v2);
+
+ if(!r) {
+ didAnyFail = true;
+ }
+
+ if(select.equals("failures") && r) {
+ continue;
+ }
+
+ if(select.equals("successes") && !r) {
+ continue;
+ }
+
+ String color = r ? "\033[92m" : "\033[91m";
+ String rs = r ? "T" : "F";
+
+ System.out.printf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line);
+ }
+
+ return didAnyFail;
+ }
+
+ public static boolean compareVersionsInFile(String filepath, String select) throws IOException {
+ List lines = new ArrayList<>();
+
+ try(BufferedReader br = new BufferedReader(new FileReader(filepath))) {
+ String line = br.readLine();
+
+ while(line != null) {
+ lines.add(line);
+ line = br.readLine();
+ }
+ }
+
+ return compareVersions(lines, select);
+ }
+
+ public static List generateVersionCompares(List versions) {
+ return IntStream.range(1, versions.size()).mapToObj(i -> {
+ String currentVersion = versions.get(i);
+ String previousVersion = versions.get(i - 1);
+ String op = compareVers(currentVersion, "=", previousVersion) ? "=" : "<";
+
+ String comparison = String.format("%s %s %s", previousVersion, op, currentVersion);
+
+ if(isUnsupportedComparison(comparison)) {
+ comparison = "# " + comparison;
+ }
+
+ return comparison;
+ }).collect(Collectors.toList());
+ }
+
+ public static List generatePackageCompares(Map> packages) {
+ return packages
+ .values()
+ .stream()
+ .map(GenerateMavenVersions::generateVersionCompares)
+ .flatMap(Collection::stream)
+ .distinct()
+ .collect(Collectors.toList());
+ }
+
+ public static String getSelectFilter() {
+ // set this to either "failures" or "successes" to only have those comparison results
+ // printed; setting it to anything else will have all comparison results printed
+ String value = System.getenv("VERSION_GENERATOR_PRINT");
+
+ if(value == null) {
+ return "failures";
+ }
+
+ return value;
+ }
+
+ public static void main(String[] args) throws IOException {
+ String outfile = "semantic/fixtures/maven-versions-generated.txt";
+ Map> packages = fetchPackageVersions();
+
+ writeToFile(outfile, generatePackageCompares(packages));
+
+ String show = getSelectFilter();
+
+ boolean didAnyFail = compareVersionsInFile(outfile, show);
+
+ if(didAnyFail) {
+ System.exit(1);
+ }
+ }
+}
diff --git a/semantic/generators/generate-alpine-versions.py b/semantic/generators/generate-alpine-versions.py
new file mode 100755
index 00000000..aac1de9c
--- /dev/null
+++ b/semantic/generators/generate-alpine-versions.py
@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+
+import atexit
+import json
+import operator
+import os
+import subprocess
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+# this requires being run on an OS with docker available to run an alpine container
+# through which apk can be invoked to compare versions natively.
+#
+# this generator will attempt to run an alpine container in the background
+# for the lifetime of the generator that will be used to exec apk; this is a lot faster
+# than running a dedicated container for each invocation, but does mean the container
+# may need to be cleaned up manually if the generator explodes in a way that prevents
+# it from stopping the container before exiting.
+#
+# this generator also uses cache to store the results of comparisons given the large
+# volume of packages and versions to compare, which is stored in the /tmp directory.
+
+# An array of version comparisons that are known to be unsupported and so
+# should be commented out in the generated fixture.
+#
+# Generally this is because the native implementation has a suspected bug
+# that causes the comparison to return incorrect results, and so supporting
+# such comparisons in the detector would in fact be wrong.
+UNSUPPORTED_COMPARISONS = []
+
+
+def is_unsupported_comparison(line):
+ return line in UNSUPPORTED_COMPARISONS
+
+
+def uncomment(line):
+ if line.startswith("#"):
+ return line[1:]
+ if line.startswith("//"):
+ return line[2:]
+ return line
+
+
+def download_alpine_db():
+ urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/Alpine/all.zip", "alpine-db.zip")
+
+
+def extract_packages_with_versions(osvs):
+ dict = {}
+
+ for osv in osvs:
+ for affected in osv['affected']:
+ if 'package' not in affected or not affected['package']['ecosystem'].startswith('Alpine'):
+ continue
+
+ package = affected['package']['name']
+
+ if package not in dict:
+ dict[package] = []
+
+ for version in affected.get('versions', []):
+ dict[package].append(AlpineVersion(version))
+
+ # deduplicate and sort the versions for each package
+ for package in dict:
+ dict[package] = sorted(list(dict.fromkeys(dict[package])))
+
+ return dict
+
+
+class AlpineVersionComparer:
+ def __init__(self, cache_path, how):
+ self.cache_path = Path(cache_path)
+ self.cache = {}
+
+ self._alpine_version = "3.10"
+ self._compare_method = how
+ self._docker_container = None
+ self._load_cache()
+
+ def _start_docker_container(self):
+ """
+ Starts the Alpine docker container for use in comparing versions using apk,
+ assigning the name of the container to `self._docker_container` if success.
+
+ If a container has already been started, this does nothing.
+ """
+
+ if self._docker_container is not None:
+ return
+
+ container_name = f"alpine-{self._alpine_version}-container"
+
+ cmd = ["docker", "run", "--rm", "--name", container_name, "-d", f"alpine:{self._alpine_version}", "tail", "-f", "/dev/null"]
+ out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ if out.returncode != 0:
+ raise Exception(f"failed to start {container_name} container: {out.stderr.decode('utf-8')}")
+ self._docker_container = container_name
+ atexit.register(self._stop_docker_container)
+
+ def _stop_docker_container(self):
+ if self._docker_container is None:
+ raise Exception(f"called to stop docker container when none was started")
+
+ cmd = ["docker", "stop", "-t", "0", self._docker_container]
+ out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ if out.returncode != 0:
+ raise Exception(f"failed to stop {self._docker_container} container: {out.stderr.decode('utf-8')}")
+
+ def _load_cache(self):
+ if self.cache_path:
+ self.cache_path.touch()
+ with open(self.cache_path, "r") as f:
+ lines = f.readlines()
+
+ for line in lines:
+ line = line.strip()
+ key, result = line.split(",")
+
+ if result == "True":
+ self.cache[key] = True
+ continue
+ if result == "False":
+ self.cache[key] = False
+ continue
+
+ print(f"ignoring invalid cache entry '{line}'")
+
+ def _save_to_cache(self, key, result):
+ self.cache[key] = result
+ if self.cache_path:
+ self.cache_path.touch()
+ with open(self.cache_path, "a") as f:
+ f.write(f"{key},{result}\n")
+
+ def _compare_command(self, a, b):
+ if self._compare_method == "run":
+ return ["docker", "run", "--rm", f"alpine:{self._alpine_version}", "apk", "version", "-t", a, b]
+
+ self._start_docker_container()
+
+ return ["docker", "exec", self._docker_container, "apk", "version", "-t", a, b]
+
+ def compare(self, a, op, b):
+ key = f"{a} {op} {b}"
+ if key in self.cache:
+ return self.cache[key]
+
+ out = subprocess.run(self._compare_command(a, b), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ if out.returncode != 0:
+ raise Exception(f"apk did not like comparing {a} {op} {b}: {out.stderr.decode('utf-8')}")
+
+ r = out.stdout.decode('utf-8').strip() == op
+ self._save_to_cache(key, r)
+ return r
+
+
+alpine_comparer = AlpineVersionComparer("/tmp/alpine-versions-generator-cache.csv", "exec")
+
+
+class AlpineVersion:
+ def __str__(self):
+ return self.version
+
+ def __hash__(self):
+ return hash(self.version)
+
+ def __init__(self, version):
+ self.version = version
+
+ def __lt__(self, other):
+ return alpine_comparer.compare(self.version, '<', other.version)
+
+ def __gt__(self, other):
+ return alpine_comparer.compare(self.version, '>', other.version)
+
+ def __eq__(self, other):
+ return alpine_comparer.compare(self.version, '=', other.version)
+
+
+def compare(v1, relate, v2):
+ ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt}
+ return ops[relate](v1, v2)
+
+
+def compare_versions(lines, select="all"):
+ has_any_failed = False
+
+ for line in lines:
+ line = line.strip()
+
+ if line == "" or line.startswith('#') or line.startswith('//'):
+ maybe_unsupported = uncomment(line).strip()
+
+ if is_unsupported_comparison(maybe_unsupported):
+ print(f"\033[96mS\033[0m: \033[93m{maybe_unsupported}\033[0m")
+ continue
+
+ v1, op, v2 = line.strip().split(" ")
+
+ r = compare(AlpineVersion(v1), op, AlpineVersion(v2))
+
+ if not r:
+ has_any_failed = r
+
+ if select == "failures" and r:
+ continue
+
+ if select == "successes" and not r:
+ continue
+
+ color = '\033[92m' if r else '\033[91m'
+ rs = "T" if r else "F"
+ print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m")
+ return has_any_failed
+
+
+def compare_versions_in_file(filepath, select="all"):
+ with open(filepath) as f:
+ lines = f.readlines()
+ return compare_versions(lines, select)
+
+
+def generate_version_compares(versions):
+ comparisons = []
+ for i, version in enumerate(versions):
+ if i == 0:
+ continue
+
+ comparison = f"{versions[i - 1]} < {version}\n"
+
+ if is_unsupported_comparison(comparison.strip()):
+ comparison = "# " + comparison
+ comparisons.append(comparison)
+ return comparisons
+
+
+def generate_package_compares(packages):
+ comparisons = []
+ for package in packages:
+ versions = packages[package]
+ comparisons.extend(generate_version_compares(versions))
+
+ # return comparisons
+ return list(dict.fromkeys(comparisons))
+
+
+def fetch_packages_versions():
+ download_alpine_db()
+ osvs = []
+
+ with zipfile.ZipFile('alpine-db.zip') as db:
+ for fname in db.namelist():
+ with db.open(fname) as osv:
+ osvs.append(json.loads(osv.read().decode('utf-8')))
+
+ return extract_packages_with_versions(osvs)
+
+
+outfile = "semantic/fixtures/alpine-versions-generated.txt"
+
+packs = fetch_packages_versions()
+with open(outfile, "w") as f:
+ f.writelines(generate_package_compares(packs))
+ f.write("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = os.environ.get("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail = compare_versions_in_file(outfile, show)
+
+if did_any_fail:
+ sys.exit(1)
diff --git a/semantic/generators/generate-cran-versions.R b/semantic/generators/generate-cran-versions.R
new file mode 100755
index 00000000..ea91fc2f
--- /dev/null
+++ b/semantic/generators/generate-cran-versions.R
@@ -0,0 +1,187 @@
+#!/usr/bin/env Rscript
+
+install.packages("jsonlite", repos = 'https://cran.r-project.org')
+
+library(utils)
+library(jsonlite)
+
+# An array of version comparisons that are known to be unsupported and so
+# should be commented out in the generated fixture.
+#
+# Generally this is because the native implementation has a suspected bug
+# that causes the comparison to return incorrect results, and so supporting
+# such comparisons in the detector would in fact be wrong.
+UNSUPPORTED_COMPARISONS <- c()
+
+download_cran_db <- function() {
+ url <- "https://osv-vulnerabilities.storage.googleapis.com/CRAN/all.zip"
+ dest <- "cran-db.zip"
+ download.file(url, dest, method = "auto")
+}
+
+extract_packages_with_versions <- function(osvs) {
+ result <- list()
+
+ for (osv in osvs) {
+ for (affected in osv$affected) {
+ if (is.null(affected["package"]) || affected$package$ecosystem != "CRAN") {
+ next
+ }
+
+ package <- affected$package$name
+
+ if (!(package %in% names(result))) {
+ result[[package]] <- list()
+ }
+
+ for (version in affected$versions) {
+ tryCatch(
+ {
+ as.package_version(version)
+ result[[package]] <- c(result[[package]], version)
+ },
+ error = function(e) {
+ cat(sprintf("skipping invalid version %s for %s\n", version, package))
+ }
+ )
+ }
+ }
+ }
+
+ # deduplicate and sort the versions for each package
+ for (package in names(result)) {
+ result[[package]] <- sort(numeric_version(unique(result[[package]])))
+ }
+
+ return(result)
+}
+
+is_unsupported_comparison <- function(line) {
+ line %in% UNSUPPORTED_COMPARISONS
+}
+
+uncomment <- function(line) {
+ if (startsWith(line, "#")) {
+ return(substr(line, 2, nchar(line)))
+ }
+ if (startsWith(line, "//")) {
+ return(substr(line, 3, nchar(line)))
+ }
+ return(line)
+}
+
+compare <- function(v1, relate, v2) {
+ ops <- list('<' = function(result) result < 0,
+ '=' = function(result) result == 0,
+ '>' = function(result) result > 0)
+
+ return(ops[[relate]](compareVersion(v1, v2)))
+}
+
+compare_versions <- function(lines, select="all") {
+ has_any_failed <- FALSE
+
+ for (line in lines) {
+ line <- trimws(line)
+
+ if (line == "" || grepl("^#", line) || grepl("^//", line)) {
+ maybe_unsupported <- trimws(uncomment(line))
+
+ if (is_unsupported_comparison(maybe_unsupported)) {
+ cat(sprintf("\033[96mS\033[0m: \033[93m%s\033[0m\n", maybe_unsupported))
+ }
+ next
+ }
+
+ parts <- strsplit(trimws(line), " ")[[1]]
+ v1 <- parts[1]
+ op <- parts[2]
+ v2 <- parts[3]
+
+ r <- compare(v1, op, v2)
+
+ if (!r) {
+ has_any_failed <- TRUE
+ }
+
+ if (select == "failures" && r) {
+ next
+ }
+
+ if (select == "successes" && !r) {
+ next
+ }
+
+ color <- ifelse(r, '\033[92m', '\033[91m')
+ rs <- ifelse(r, "T", "F")
+ cat(sprintf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line))
+ }
+ return(has_any_failed)
+}
+
+compare_versions_in_file <- function(filepath, select="all") {
+ lines <- readLines(filepath)
+ return(compare_versions(lines, select))
+}
+
+generate_version_compares <- function(versions) {
+ comparisons <- character()
+
+ for (i in seq_along(versions)) {
+ if (i == 1) {
+ next
+ }
+
+ comparison <- sprintf("%s < %s", versions[i - 1], versions[i])
+
+ if (is_unsupported_comparison(trimws(comparison))) {
+ comparison <- paste("#", comparison)
+ }
+
+ comparisons <- c(comparisons, comparison)
+ }
+
+ return(comparisons)
+}
+
+generate_package_compares <- function(packages) {
+ comparisons <- character()
+
+ for (package in names(packages)) {
+ versions <- packages[[package]]
+ comparisons <- c(comparisons, generate_version_compares(versions))
+ }
+
+ # return unique comparisons
+ return(unique(comparisons))
+}
+
+fetch_packages_versions <- function() {
+ download_cran_db()
+ osvs <- list()
+
+ with_zip <- unzip("cran-db.zip", list = TRUE)
+
+ for (fname in with_zip$Name) {
+ osv <- jsonlite::fromJSON(unzip("cran-db.zip", files = fname, exdir = tempdir()), simplifyDataFrame = FALSE)
+ osvs <- c(osvs, list(osv))
+ }
+
+ return(extract_packages_with_versions(osvs))
+}
+
+outfile <- "semantic/fixtures/cran-versions-generated.txt"
+
+packs <- fetch_packages_versions()
+writeLines(generate_package_compares(packs), outfile, sep = "\n")
+cat("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show <- Sys.getenv("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail <- compare_versions_in_file(outfile, show)
+
+if (did_any_fail) {
+ q(status = 1)
+}
diff --git a/semantic/generators/generate-debian-versions.py b/semantic/generators/generate-debian-versions.py
new file mode 100755
index 00000000..c3af8a3b
--- /dev/null
+++ b/semantic/generators/generate-debian-versions.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+
+import json
+import operator
+import os
+import subprocess
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+# this requires being run on an OS that has a version of "dpkg" which supports the
+# "--compare-versions" option; also make sure to consider the version of dpkg being
+# used in case there are changes to the comparing logic (last run with 1.19.7).
+#
+# also note that because of the large amount of versions being used there is
+# significant overhead in having to use a subprocess, so this generator caches
+# the results of said subprocess calls; a typical no-cache run takes about 5+
+# minutes whereas with the cache it only takes seconds.
+
+# An array of version comparisons that are known to be unsupported and so
+# should be commented out in the generated fixture.
+#
+# Generally this is because the native implementation has a suspected bug
+# that causes the comparison to return incorrect results, and so supporting
+# such comparisons in the detector would in fact be wrong.
+UNSUPPORTED_COMPARISONS = []
+
+
+def is_unsupported_comparison(line):
+ return line in UNSUPPORTED_COMPARISONS
+
+
+def uncomment(line):
+ if line.startswith("#"):
+ return line[1:]
+ if line.startswith("//"):
+ return line[2:]
+ return line
+
+
+def download_debian_db():
+ urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip", "debian-db.zip")
+
+
+def extract_packages_with_versions(osvs):
+ dict = {}
+
+ for osv in osvs:
+ for affected in osv['affected']:
+ if 'package' not in affected or not affected['package']['ecosystem'].startswith('Debian'):
+ continue
+
+ package = affected['package']['name']
+
+ if package not in dict:
+ dict[package] = []
+
+ for version in affected.get('versions', []):
+ dict[package].append(DebianVersion(version))
+
+ # deduplicate and sort the versions for each package
+ for package in dict:
+ dict[package] = sorted(list(dict.fromkeys(dict[package])))
+
+ return dict
+
+
+class DebianVersionComparer:
+ def __init__(self, cache_path):
+ self.cache_path = Path(cache_path)
+ self.cache = {}
+
+ self._load_cache()
+
+ def _load_cache(self):
+ if self.cache_path:
+ self.cache_path.touch()
+ with open(self.cache_path, "r") as f:
+ lines = f.readlines()
+
+ for line in lines:
+ line = line.strip()
+ key, result = line.split(",")
+
+ if result == "True":
+ self.cache[key] = True
+ continue
+ if result == "False":
+ self.cache[key] = False
+ continue
+
+ print(f"ignoring invalid cache entry '{line}'")
+
+ def _save_to_cache(self, key, result):
+ self.cache[key] = result
+ if self.cache_path:
+ self.cache_path.touch()
+ with open(self.cache_path, "a") as f:
+ f.write(f"{key},{result}\n")
+
+ def compare(self, a, op, b):
+ key = f"{a} {op} {b}"
+ if key in self.cache:
+ return self.cache[key]
+
+ cmd = ["dpkg", "--compare-versions", a, op, b]
+ out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ if out.stdout:
+ print(out.stdout.decode('utf-8'))
+ if out.stderr:
+ print(out.stderr.decode('utf-8'))
+
+ r = out.returncode == 0
+ self._save_to_cache(key, r)
+ return r
+
+
+debian_comparer = DebianVersionComparer("/tmp/debian-versions-generator-cache.csv")
+
+
+class DebianVersion:
+ def __str__(self):
+ return self.version
+
+ def __hash__(self):
+ return hash(self.version)
+
+ def __init__(self, version):
+ self.version = version
+
+ def __lt__(self, other):
+ return debian_comparer.compare(self.version, 'lt', other.version)
+
+ def __gt__(self, other):
+ return debian_comparer.compare(self.version, 'gt', other.version)
+
+ def __eq__(self, other):
+ return debian_comparer.compare(self.version, 'eq', other.version)
+
+
+def compare(v1, relate, v2):
+ ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt}
+ return ops[relate](v1, v2)
+
+
+def compare_versions(lines, select="all"):
+ has_any_failed = False
+
+ for line in lines:
+ line = line.strip()
+
+ if line == "" or line.startswith('#') or line.startswith('//'):
+ maybe_unsupported = uncomment(line).strip()
+
+ if is_unsupported_comparison(maybe_unsupported):
+ print(f"\033[96mS\033[0m: \033[93m{maybe_unsupported}\033[0m")
+ continue
+
+ v1, op, v2 = line.strip().split(" ")
+
+ r = compare(DebianVersion(v1), op, DebianVersion(v2))
+
+ if not r:
+ has_any_failed = r
+
+ if select == "failures" and r:
+ continue
+
+ if select == "successes" and not r:
+ continue
+
+ color = '\033[92m' if r else '\033[91m'
+ rs = "T" if r else "F"
+ print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m")
+ return has_any_failed
+
+
+def compare_versions_in_file(filepath, select="all"):
+ with open(filepath) as f:
+ lines = f.readlines()
+ return compare_versions(lines, select)
+
+
+def generate_version_compares(versions):
+ comparisons = []
+ for i, version in enumerate(versions):
+ if i == 0:
+ continue
+
+ comparison = f"{versions[i - 1]} < {version}\n"
+
+ if is_unsupported_comparison(comparison.strip()):
+ comparison = "# " + comparison
+ comparisons.append(comparison)
+ return comparisons
+
+
+def generate_package_compares(packages):
+ comparisons = []
+ for package in packages:
+ versions = packages[package]
+ comparisons.extend(generate_version_compares(versions))
+
+ # return comparisons
+ return list(dict.fromkeys(comparisons))
+
+
+def fetch_packages_versions():
+ download_debian_db()
+ osvs = []
+
+ with zipfile.ZipFile('debian-db.zip') as db:
+ for fname in db.namelist():
+ with db.open(fname) as osv:
+ osvs.append(json.loads(osv.read().decode('utf-8')))
+
+ return extract_packages_with_versions(osvs)
+
+
+outfile = "semantic/fixtures/debian-versions-generated.txt"
+
+packs = fetch_packages_versions()
+with open(outfile, "w") as f:
+ f.writelines(generate_package_compares(packs))
+ f.write("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = os.environ.get("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail = compare_versions_in_file(outfile, show)
+
+if did_any_fail:
+ sys.exit(1)
diff --git a/semantic/generators/generate-packagist-versions.php b/semantic/generators/generate-packagist-versions.php
new file mode 100755
index 00000000..fbe03d84
--- /dev/null
+++ b/semantic/generators/generate-packagist-versions.php
@@ -0,0 +1,231 @@
+open($path, ZipArchive::RDONLY) === false) {
+ throw new RuntimeException('failed to read zip archive');
+ }
+
+ return $zip;
+}
+
+/**
+ * @throws JsonException
+ * @throws RuntimeException
+ */
+function fetchPackageVersions(): array
+{
+ $dbPath = downloadPackagistDb();
+ $dbZip = openDbZip($dbPath);
+
+ $osvs = [];
+
+ for ($i = 0; $i < $dbZip->numFiles; $i++) {
+ $file = $dbZip->getFromIndex($i);
+
+ if ($file === false) {
+ throw new RuntimeException('failed to read a file from db zip');
+ }
+
+ $osvs[] = json_decode($file, true, 512, JSON_THROW_ON_ERROR);
+ }
+
+ $packages = [];
+
+ foreach ($osvs as $osv) {
+ foreach ($osv['affected'] as $affected) {
+ if (!isset($affected['package']) || $affected['package']['ecosystem'] !== 'Packagist') {
+ continue;
+ }
+
+ $package = $affected['package']['name'];
+
+ if (!isset($packages[$package])) {
+ $packages[$package] = [];
+ }
+
+ if (empty($affected['versions'])) {
+ continue;
+ }
+
+ foreach ($affected['versions'] as $version) {
+ $packages[$package][] = $version;
+ }
+ }
+ }
+
+ return array_map(static function ($versions) {
+ $uniq = array_unique($versions);
+ usort($uniq, static fn($a, $b) => version_compare(ltrim($a, "vV"), ltrim($b, "vV")));
+
+ return $uniq;
+ }, $packages);
+}
+
+/**
+ * Normalizes the previous version such that it will compare "correctly" to the current version,
+ * by ensuring that they both have the same "v" prefix (or lack of).
+ *
+ * Whether the "v" prefix is present on the normalized previous version depends on
+ * its presences in the current version; this ensure we will have _some_ versions that
+ * do have the "v" prefix, rather than it being present on _none_ or _all_ versions.
+ *
+ * @param string $currentVersion
+ * @param string $previousVersion
+ *
+ * @return string
+ */
+function normalizePrevVersion(string $currentVersion, string $previousVersion): string
+{
+ if (str_starts_with($currentVersion, "v")) {
+ $previousVersion = ltrim($previousVersion, "vV");
+
+ return "v$previousVersion";
+ }
+
+ if (str_starts_with($currentVersion, "V")) {
+ $previousVersion = ltrim($previousVersion, "vV");
+
+ return "V$previousVersion";
+ }
+
+ return ltrim($previousVersion, "vV");
+}
+
+function generateVersionCompares(array $versions): array
+{
+ $comparisons = [];
+
+ foreach ($versions as $index => $version) {
+ if ($index === 0) {
+ continue;
+ }
+
+ $prevVersion = normalizePrevVersion($version, $versions[$index - 1]);
+ $op = version_compare($prevVersion, $version) === 0 ? "=" : "<";
+
+ $comparison = "$prevVersion $op $version";
+
+ if (isUnsupportedComparison($comparison)) {
+ $comparison = "# $comparison";
+ }
+
+ $comparisons[] = $comparison;
+ }
+
+ return $comparisons;
+}
+
+function generatePackageCompares(array $packages): array
+{
+ $comparisons = [];
+
+ foreach ($packages as $versions) {
+ $comparisons[] = generateVersionCompares($versions);
+ }
+
+ return array_merge(...$comparisons);
+}
+
+function compareVersions(array $lines, string $select = "all"): bool
+{
+ $hasAnyFailed = false;
+
+ foreach ($lines as $line) {
+ $line = trim($line);
+
+ if (empty($line) || str_starts_with($line, "#") || str_starts_with($line, "//")) {
+ $maybeUnsupported = trim(uncomment($line));
+
+ if (isUnsupportedComparison($maybeUnsupported)) {
+ echo "\033[96mS\033[0m: \033[93m$maybeUnsupported\033[0m\n";
+ }
+
+ continue;
+ }
+
+ [$v1, $op, $v2] = explode(" ", $line);
+
+ $r = version_compare($v1, $v2, $op);
+
+ if (!$r) {
+ $hasAnyFailed = true;
+ }
+
+ if ($select === "failures" && $r === true) {
+ continue;
+ }
+
+ if ($select === "successes" && $r !== true) {
+ continue;
+ }
+
+ $color = $r ? "\033[92m" : "\033[91m";
+ $rs = $r ? "T" : "F";
+ echo "$color$rs\033[0m: \033[93m$line\033[0m\n";
+ }
+
+ return $hasAnyFailed;
+}
+
+$outfile = "semantic/fixtures/packagist-versions-generated.txt";
+
+/** @noinspection PhpUnhandledExceptionInspection */
+$packages = fetchPackageVersions();
+
+file_put_contents($outfile, implode("\n", array_unique(generatePackageCompares($packages))) . "\n");
+
+// set this to either "failures" or "successes" to only have those comparison results
+// printed; setting it to anything else will have all comparison results printed
+$show = getenv("VERSION_GENERATOR_PRINT") ?: "failures";
+
+$didAnyFail = compareVersions(explode("\n", file_get_contents($outfile)), $show);
+
+if ($didAnyFail === true) {
+ exit(1);
+}
diff --git a/semantic/generators/generate-pypi-versions.py b/semantic/generators/generate-pypi-versions.py
new file mode 100755
index 00000000..f14a2d85
--- /dev/null
+++ b/semantic/generators/generate-pypi-versions.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+import json
+import operator
+import os
+import packaging.version
+import sys
+import urllib.request
+import zipfile
+
+# this requires you run "pip install packaging" - have to be careful about versions too
+# because of the "legacy version" stuff
+
+# An array of version comparisons that are known to be unsupported and so
+# should be commented out in the generated fixture.
+#
+# Generally this is because the native implementation has a suspected bug
+# that causes the comparison to return incorrect results, and so supporting
+# such comparisons in the detector would in fact be wrong.
+UNSUPPORTED_COMPARISONS = []
+
+
+def is_unsupported_comparison(line):
+ return line in UNSUPPORTED_COMPARISONS
+
+
+def uncomment(line):
+ if line.startswith("#"):
+ return line[1:]
+ if line.startswith("//"):
+ return line[2:]
+ return line
+
+
+def download_pypi_db():
+ urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip", "pypi-db.zip")
+
+
+def extract_packages_with_versions(osvs):
+ dict = {}
+
+ for osv in osvs:
+ for affected in osv['affected']:
+ if 'package' not in affected or affected['package']['ecosystem'] != 'PyPI':
+ continue
+
+ package = affected['package']['name']
+
+ if package not in dict:
+ dict[package] = []
+
+ for version in affected.get('versions', []):
+ try:
+ dict[package].append(packaging.version.parse(version))
+ except packaging.version.InvalidVersion:
+ print(f"skipping invalid version {version} for {package}")
+
+ # deduplicate and sort the versions for each package
+ for package in dict:
+ dict[package] = sorted(list(dict.fromkeys(dict[package])))
+
+ return dict
+
+
+def compare(v1, relate, v2):
+ ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt}
+ return ops[relate](v1, v2)
+
+
+def compare_versions(lines, select="all"):
+ has_any_failed = False
+
+ for line in lines:
+ line = line.strip()
+
+ if line == "" or line.startswith('#') or line.startswith('//'):
+ maybe_unsupported = uncomment(line).strip()
+
+ if is_unsupported_comparison(maybe_unsupported):
+ print(f"\033[96mS\033[0m: \033[93m{maybe_unsupported}\033[0m")
+ continue
+
+ v1, op, v2 = line.strip().split(" ")
+
+ r = compare(packaging.version.parse(v1), op, packaging.version.parse(v2))
+
+ if not r:
+ has_any_failed = True
+
+ if select == "failures" and r:
+ continue
+
+ if select == "successes" and not r:
+ continue
+
+ color = '\033[92m' if r else '\033[91m'
+ rs = "T" if r else "F"
+ print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m")
+ return has_any_failed
+
+
+def compare_versions_in_file(filepath, select="all"):
+ with open(filepath) as f:
+ lines = f.readlines()
+ return compare_versions(lines, select)
+
+
+def generate_version_compares(versions):
+ comparisons = []
+ for i, version in enumerate(versions):
+ if i == 0:
+ continue
+
+ comparison = f"{versions[i - 1]} < {version}\n"
+
+ if is_unsupported_comparison(comparison.strip()):
+ comparison = "# " + comparison
+ comparisons.append(comparison)
+ return comparisons
+
+
+def generate_package_compares(packages):
+ comparisons = []
+ for package in packages:
+ versions = packages[package]
+ comparisons.extend(generate_version_compares(versions))
+
+ # return comparisons
+ return list(dict.fromkeys(comparisons))
+
+
+def fetch_packages_versions():
+ download_pypi_db()
+ osvs = []
+
+ with zipfile.ZipFile('pypi-db.zip') as db:
+ for fname in db.namelist():
+ with db.open(fname) as osv:
+ osvs.append(json.loads(osv.read().decode('utf-8')))
+
+ return extract_packages_with_versions(osvs)
+
+
+outfile = "semantic/fixtures/pypi-versions-generated.txt"
+
+packs = fetch_packages_versions()
+with open(outfile, "w") as f:
+ f.writelines(generate_package_compares(packs))
+ f.write("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = os.environ.get("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail = compare_versions_in_file(outfile, show)
+
+if did_any_fail:
+ sys.exit(1)
diff --git a/semantic/generators/generate-rubygems-versions.rb b/semantic/generators/generate-rubygems-versions.rb
new file mode 100755
index 00000000..b638c88b
--- /dev/null
+++ b/semantic/generators/generate-rubygems-versions.rb
@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+require "rubygems/version"
+require "open-uri"
+require "json"
+require "zip"
+
+# An array of version comparisons that are known to be unsupported and so
+# should be commented out in the generated fixture.
+#
+# Generally this is because the native implementation has a suspected bug
+# that causes the comparison to return incorrect results, and so supporting
+# such comparisons in the detector would in fact be wrong.
+#
+# @type [Array]
+UNSUPPORTED_COMPARISONS = [].freeze
+
+# @param [String] line
+# @return [Boolean]
+def is_unsupported_comparison?(line)
+ UNSUPPORTED_COMPARISONS.include? line
+end
+
+# @param [String] line
+# @return [String]
+def uncomment(line)
+ line.sub(%r{^#|//}, "")
+end
+
+def download_rubygems_db
+ URI.open("https://osv-vulnerabilities.storage.googleapis.com/RubyGems/all.zip") do |zip|
+ File.binwrite("rubygems-db.zip", zip.read)
+ end
+end
+
+def extract_packages_with_versions(osvs)
+ packages = {}
+
+ osvs.each do |osv|
+ osv["affected"].each do |affected|
+ next unless affected.dig("package", "ecosystem") == "RubyGems"
+
+ package = affected["package"]["name"]
+
+ packages[package] ||= []
+ affected.fetch("versions", []).each do |version|
+ packages[package] << Gem::Version.new(version)
+ end
+ end
+ end
+
+ packages.transform_values { |v| v.uniq.sort }
+end
+
+def compare_version(v1, op, v2)
+ op = "==" if op == "="
+
+ Gem::Version.new(v1).method(op).call(Gem::Version.new(v2))
+end
+
+# @param [Array] lines
+# @return [Boolean]
+def compare_versions(lines, select = :all)
+ has_any_failed = false
+
+ lines.each do |line|
+ line = line.strip
+
+ if line.empty? || line.start_with?("#") || line.start_with?("//")
+ maybe_unsupported = uncomment(line).strip
+
+ puts "\033[96mS\033[0m: \033[93m#{maybe_unsupported}\033[0m" if is_unsupported_comparison?(maybe_unsupported)
+
+ next
+ end
+
+ parts = line.split
+ v1 = parts[0]
+ op = parts[1]
+ v2 = parts[2]
+
+ r = compare_version(v1, op, v2)
+
+ has_any_failed = true unless r
+
+ next if select == :failures && r == true
+ next if select == :successes && r != true
+
+ color = r ? "\033[92m" : "\033[91m"
+ rs = r ? "T" : "F"
+ puts "#{color}#{rs}\033[0m: \033[93m#{line}\033[0m"
+ end
+
+ has_any_failed
+end
+
+def compare_versions_in_file(filepath, select = :all)
+ compare_versions(File.readlines(filepath), select)
+end
+
+def generate_version_compares(versions)
+ comparisons = []
+
+ versions.each_with_index do |version, i|
+ next if i == 0
+
+ op = "<"
+ op = "=" if versions[i - 1] == version
+
+ comparison = "#{versions[i - 1]} #{op} #{version}"
+ comparison = "# #{comparison}" if is_unsupported_comparison?(comparison)
+
+ comparisons << comparison
+ end
+
+ comparisons
+end
+
+def generate_package_compares(packages)
+ comparisons = []
+
+ packages.each_value { |versions| comparisons.concat(generate_version_compares(versions)) }
+
+ comparisons
+end
+
+def fetch_packages_versions
+ download_rubygems_db
+
+ osvs = Zip::File.open("rubygems-db.zip").map { |f| JSON.parse(f.get_input_stream.read) }
+
+ extract_packages_with_versions(osvs)
+end
+
+outfile = "semantic/fixtures/rubygems-versions-generated.txt"
+
+packs = fetch_packages_versions
+
+File.write(outfile, "#{generate_package_compares(packs).uniq.join("\n")}\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = ENV.fetch("VERSION_GENERATOR_PRINT", :failures).to_sym
+
+did_any_fail = compare_versions_in_file(outfile, show)
+
+exit(1) if did_any_fail