diff --git a/semantic/generators/GenerateMavenVersions.java b/semantic/generators/GenerateMavenVersions.java new file mode 100755 index 00000000..f0816852 --- /dev/null +++ b/semantic/generators/GenerateMavenVersions.java @@ -0,0 +1,291 @@ +import org.apache.maven.artifact.versioning.ComparableVersion; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.*; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Script for generating a list of maven version comparison fixtures based off + * every version mentioned in the OSV Maven database, sorted using the native + * Maven implementation. + *

+ * To run this, you need to ensure copies of the following libraries are present + * on the class path: + * + *

+ * The easiest way to do this is by putting the jars into a lib subfolder and then running: + * + * java -cp generators/lib/* generators/GenerateMavenVersions.java + * + */ +public class GenerateMavenVersions { + /** + * An array of version comparisons that are known to be unsupported and so + * should be commented out in the generated fixture. + *

+ * Generally this is because the native implementation has a suspected bug + * that causes the comparison to return incorrect results, and so supporting + * such comparisons in the detector would in fact be wrong. + */ + private static final String[] UNSUPPORTED_COMPARISONS = { + "0.0.0-2021-07-06T00-28-13-573087f7 < 0.0.0-2021-07-06T01-14-42-efe42242", + "0.0.0-2021-12-06T00-08-57-89a33731 < 0.0.0-2021-12-06T01-21-56-e3888760", + "0.0.0-2022-02-01T00-45-53-0300684a < 0.0.0-2022-02-01T05-45-16-7258ece0", + "0.0.0-2022-02-28T00-18-39-7fe0d845 < 0.0.0-2022-02-28T04-15-47-83c97ebe", + "0.0.0-2022-04-29T00-08-11-7086a3ec < 0.0.0-2022-04-29T01-20-09-b424f986", + "0.0.0-2022-06-14T00-21-33-f21869a7 < 0.0.0-2022-06-14T02-56-29-1db980e0", + "0.0.0-2022-08-16T00-14-19-aeae3dc3 < 0.0.0-2022-08-16T10-34-26-7a56f709", + "0.0.0-2022-08-22T00-46-32-4652d3db < 0.0.0-2022-08-22T06-46-40-e7409ac5", + "0.0.0-2022-10-31T00-42-12-322ba6b9 < 0.0.0-2022-10-31T01-23-06-c6652489", + "0.0.0-2022-10-31T07-00-43-71eccd49 < 0.0.0-2022-10-31T07-05-43-97874976", + "0.0.0-2022-12-01T00-02-29-fe8d6705 < 0.0.0-2022-12-01T01-56-22-5b442198", + "0.0.0-2022-12-18T00-44-34-a222f475 < 0.0.0-2022-12-18T01-45-19-fec81751", + "0.0.0-2023-03-20T00-52-15-4b4c0e7 < 0.0.0-2023-03-20T01-49-44-80e3135" + }; + + public static boolean isUnsupportedComparison(String line) { + return Arrays.stream(UNSUPPORTED_COMPARISONS).anyMatch(line::equals); + } + + public static String uncomment(String line) { + if(line.startsWith("#")) { + return line.substring(1); + } + + if(line.startsWith("//")) { + return line.substring(2); + } + + return line; + } + + public static String downloadMavenDb() throws IOException { + URL website = new URL("https://osv-vulnerabilities.storage.googleapis.com/Maven/all.zip"); + String file = "./maven-db.zip"; + + ReadableByteChannel rbc = Channels.newChannel(website.openStream()); + + try(FileOutputStream fos = new FileOutputStream(file)) { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + } + + return file; + } + + public static Map> fetchPackageVersions() throws IOException { + String dbPath = downloadMavenDb(); + List osvs = loadOSVs(dbPath); + + Map> packages = new HashMap<>(); + + osvs.forEach(osv -> osv.getJSONArray("affected").forEach(aff -> { + JSONObject affected = (JSONObject) aff; + + if(!affected.has("package") || affected.getJSONObject("package").getString("ecosystem").equals("Maven")) { + return; + } + + String pkgName = affected.getJSONObject("package").getString("name"); + + if(!affected.has("versions")) { + return; + } + JSONArray versions = affected.getJSONArray("versions"); + + packages.putIfAbsent(pkgName, new ArrayList<>()); + + if(versions.isEmpty()) { + return; + } + + versions.forEach(version -> packages.get(pkgName).add((String) version)); + })); + + packages.forEach((key, _ignore) -> packages.put( + key, + packages.get(key) + .stream() + .distinct() + .sorted(Comparator.comparing(ComparableVersion::new)) + .collect(Collectors.toList()) + )); + + return packages; + } + + public static List loadOSVs(String pathToDbZip) throws IOException { + List osvs = new ArrayList<>(); + + try(ZipFile zipFile = new ZipFile(pathToDbZip)) { + Enumeration entries = zipFile.entries(); + + while(entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + InputStream stream = zipFile.getInputStream(entry); + + BufferedReader streamReader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); + StringBuilder responseStrBuilder = new StringBuilder(); + + String inputStr; + while((inputStr = streamReader.readLine()) != null) { + responseStrBuilder.append(inputStr); + } + osvs.add(new JSONObject(responseStrBuilder.toString())); + } + } + + return osvs; + } + + public static void writeToFile(String outfile, List lines) throws IOException { + try(PrintWriter writer = new PrintWriter(outfile, StandardCharsets.UTF_8)) { + lines.forEach(writer::println); + } + } + + public static boolean compareVers(String version1, String op, String version2) { + ComparableVersion v1 = new ComparableVersion(version1); + ComparableVersion v2 = new ComparableVersion(version2); + + int r = v1.compareTo(v2); + + if(op.equals("=")) { + return r == 0; + } + + if(op.equals("<")) { + return r < 0; + } + + if(op.equals(">")) { + return r > 0; + } + + throw new RuntimeException("unsupported comparison operator " + op); + } + + public static boolean compareVersions(List lines, String select) { + boolean didAnyFail = false; + + for(String line : lines) { + line = line.trim(); + + if(line.isEmpty() || line.startsWith("#") || line.startsWith("//")) { + String maybeUnsupported = uncomment(line).trim(); + + if(isUnsupportedComparison(maybeUnsupported)) { + System.out.printf("\033[96mS\033[0m: \033[93m%s\033[0m\n", maybeUnsupported); + } + + continue; + } + + String[] parts = line.split(" "); + String v1 = parts[0]; + String op = parts[1]; + String v2 = parts[2]; + + boolean r = compareVers(v1, op, v2); + + if(!r) { + didAnyFail = true; + } + + if(select.equals("failures") && r) { + continue; + } + + if(select.equals("successes") && !r) { + continue; + } + + String color = r ? "\033[92m" : "\033[91m"; + String rs = r ? "T" : "F"; + + System.out.printf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line); + } + + return didAnyFail; + } + + public static boolean compareVersionsInFile(String filepath, String select) throws IOException { + List lines = new ArrayList<>(); + + try(BufferedReader br = new BufferedReader(new FileReader(filepath))) { + String line = br.readLine(); + + while(line != null) { + lines.add(line); + line = br.readLine(); + } + } + + return compareVersions(lines, select); + } + + public static List generateVersionCompares(List versions) { + return IntStream.range(1, versions.size()).mapToObj(i -> { + String currentVersion = versions.get(i); + String previousVersion = versions.get(i - 1); + String op = compareVers(currentVersion, "=", previousVersion) ? "=" : "<"; + + String comparison = String.format("%s %s %s", previousVersion, op, currentVersion); + + if(isUnsupportedComparison(comparison)) { + comparison = "# " + comparison; + } + + return comparison; + }).collect(Collectors.toList()); + } + + public static List generatePackageCompares(Map> packages) { + return packages + .values() + .stream() + .map(GenerateMavenVersions::generateVersionCompares) + .flatMap(Collection::stream) + .distinct() + .collect(Collectors.toList()); + } + + public static String getSelectFilter() { + // set this to either "failures" or "successes" to only have those comparison results + // printed; setting it to anything else will have all comparison results printed + String value = System.getenv("VERSION_GENERATOR_PRINT"); + + if(value == null) { + return "failures"; + } + + return value; + } + + public static void main(String[] args) throws IOException { + String outfile = "semantic/fixtures/maven-versions-generated.txt"; + Map> packages = fetchPackageVersions(); + + writeToFile(outfile, generatePackageCompares(packages)); + + String show = getSelectFilter(); + + boolean didAnyFail = compareVersionsInFile(outfile, show); + + if(didAnyFail) { + System.exit(1); + } + } +} diff --git a/semantic/generators/generate-alpine-versions.py b/semantic/generators/generate-alpine-versions.py new file mode 100755 index 00000000..aac1de9c --- /dev/null +++ b/semantic/generators/generate-alpine-versions.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 + +import atexit +import json +import operator +import os +import subprocess +import sys +import urllib.request +import zipfile +from pathlib import Path + +# this requires being run on an OS with docker available to run an alpine container +# through which apk can be invoked to compare versions natively. +# +# this generator will attempt to run an alpine container in the background +# for the lifetime of the generator that will be used to exec apk; this is a lot faster +# than running a dedicated container for each invocation, but does mean the container +# may need to be cleaned up manually if the generator explodes in a way that prevents +# it from stopping the container before exiting. +# +# this generator also uses cache to store the results of comparisons given the large +# volume of packages and versions to compare, which is stored in the /tmp directory. + +# An array of version comparisons that are known to be unsupported and so +# should be commented out in the generated fixture. +# +# Generally this is because the native implementation has a suspected bug +# that causes the comparison to return incorrect results, and so supporting +# such comparisons in the detector would in fact be wrong. +UNSUPPORTED_COMPARISONS = [] + + +def is_unsupported_comparison(line): + return line in UNSUPPORTED_COMPARISONS + + +def uncomment(line): + if line.startswith("#"): + return line[1:] + if line.startswith("//"): + return line[2:] + return line + + +def download_alpine_db(): + urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/Alpine/all.zip", "alpine-db.zip") + + +def extract_packages_with_versions(osvs): + dict = {} + + for osv in osvs: + for affected in osv['affected']: + if 'package' not in affected or not affected['package']['ecosystem'].startswith('Alpine'): + continue + + package = affected['package']['name'] + + if package not in dict: + dict[package] = [] + + for version in affected.get('versions', []): + dict[package].append(AlpineVersion(version)) + + # deduplicate and sort the versions for each package + for package in dict: + dict[package] = sorted(list(dict.fromkeys(dict[package]))) + + return dict + + +class AlpineVersionComparer: + def __init__(self, cache_path, how): + self.cache_path = Path(cache_path) + self.cache = {} + + self._alpine_version = "3.10" + self._compare_method = how + self._docker_container = None + self._load_cache() + + def _start_docker_container(self): + """ + Starts the Alpine docker container for use in comparing versions using apk, + assigning the name of the container to `self._docker_container` if success. + + If a container has already been started, this does nothing. + """ + + if self._docker_container is not None: + return + + container_name = f"alpine-{self._alpine_version}-container" + + cmd = ["docker", "run", "--rm", "--name", container_name, "-d", f"alpine:{self._alpine_version}", "tail", "-f", "/dev/null"] + out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if out.returncode != 0: + raise Exception(f"failed to start {container_name} container: {out.stderr.decode('utf-8')}") + self._docker_container = container_name + atexit.register(self._stop_docker_container) + + def _stop_docker_container(self): + if self._docker_container is None: + raise Exception(f"called to stop docker container when none was started") + + cmd = ["docker", "stop", "-t", "0", self._docker_container] + out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if out.returncode != 0: + raise Exception(f"failed to stop {self._docker_container} container: {out.stderr.decode('utf-8')}") + + def _load_cache(self): + if self.cache_path: + self.cache_path.touch() + with open(self.cache_path, "r") as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + key, result = line.split(",") + + if result == "True": + self.cache[key] = True + continue + if result == "False": + self.cache[key] = False + continue + + print(f"ignoring invalid cache entry '{line}'") + + def _save_to_cache(self, key, result): + self.cache[key] = result + if self.cache_path: + self.cache_path.touch() + with open(self.cache_path, "a") as f: + f.write(f"{key},{result}\n") + + def _compare_command(self, a, b): + if self._compare_method == "run": + return ["docker", "run", "--rm", f"alpine:{self._alpine_version}", "apk", "version", "-t", a, b] + + self._start_docker_container() + + return ["docker", "exec", self._docker_container, "apk", "version", "-t", a, b] + + def compare(self, a, op, b): + key = f"{a} {op} {b}" + if key in self.cache: + return self.cache[key] + + out = subprocess.run(self._compare_command(a, b), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if out.returncode != 0: + raise Exception(f"apk did not like comparing {a} {op} {b}: {out.stderr.decode('utf-8')}") + + r = out.stdout.decode('utf-8').strip() == op + self._save_to_cache(key, r) + return r + + +alpine_comparer = AlpineVersionComparer("/tmp/alpine-versions-generator-cache.csv", "exec") + + +class AlpineVersion: + def __str__(self): + return self.version + + def __hash__(self): + return hash(self.version) + + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return alpine_comparer.compare(self.version, '<', other.version) + + def __gt__(self, other): + return alpine_comparer.compare(self.version, '>', other.version) + + def __eq__(self, other): + return alpine_comparer.compare(self.version, '=', other.version) + + +def compare(v1, relate, v2): + ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt} + return ops[relate](v1, v2) + + +def compare_versions(lines, select="all"): + has_any_failed = False + + for line in lines: + line = line.strip() + + if line == "" or line.startswith('#') or line.startswith('//'): + maybe_unsupported = uncomment(line).strip() + + if is_unsupported_comparison(maybe_unsupported): + print(f"\033[96mS\033[0m: \033[93m{maybe_unsupported}\033[0m") + continue + + v1, op, v2 = line.strip().split(" ") + + r = compare(AlpineVersion(v1), op, AlpineVersion(v2)) + + if not r: + has_any_failed = r + + if select == "failures" and r: + continue + + if select == "successes" and not r: + continue + + color = '\033[92m' if r else '\033[91m' + rs = "T" if r else "F" + print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m") + return has_any_failed + + +def compare_versions_in_file(filepath, select="all"): + with open(filepath) as f: + lines = f.readlines() + return compare_versions(lines, select) + + +def generate_version_compares(versions): + comparisons = [] + for i, version in enumerate(versions): + if i == 0: + continue + + comparison = f"{versions[i - 1]} < {version}\n" + + if is_unsupported_comparison(comparison.strip()): + comparison = "# " + comparison + comparisons.append(comparison) + return comparisons + + +def generate_package_compares(packages): + comparisons = [] + for package in packages: + versions = packages[package] + comparisons.extend(generate_version_compares(versions)) + + # return comparisons + return list(dict.fromkeys(comparisons)) + + +def fetch_packages_versions(): + download_alpine_db() + osvs = [] + + with zipfile.ZipFile('alpine-db.zip') as db: + for fname in db.namelist(): + with db.open(fname) as osv: + osvs.append(json.loads(osv.read().decode('utf-8'))) + + return extract_packages_with_versions(osvs) + + +outfile = "semantic/fixtures/alpine-versions-generated.txt" + +packs = fetch_packages_versions() +with open(outfile, "w") as f: + f.writelines(generate_package_compares(packs)) + f.write("\n") + +# set this to either "failures" or "successes" to only have those comparison results +# printed; setting it to anything else will have all comparison results printed +show = os.environ.get("VERSION_GENERATOR_PRINT", "failures") + +did_any_fail = compare_versions_in_file(outfile, show) + +if did_any_fail: + sys.exit(1) diff --git a/semantic/generators/generate-cran-versions.R b/semantic/generators/generate-cran-versions.R new file mode 100755 index 00000000..ea91fc2f --- /dev/null +++ b/semantic/generators/generate-cran-versions.R @@ -0,0 +1,187 @@ +#!/usr/bin/env Rscript + +install.packages("jsonlite", repos = 'https://cran.r-project.org') + +library(utils) +library(jsonlite) + +# An array of version comparisons that are known to be unsupported and so +# should be commented out in the generated fixture. +# +# Generally this is because the native implementation has a suspected bug +# that causes the comparison to return incorrect results, and so supporting +# such comparisons in the detector would in fact be wrong. +UNSUPPORTED_COMPARISONS <- c() + +download_cran_db <- function() { + url <- "https://osv-vulnerabilities.storage.googleapis.com/CRAN/all.zip" + dest <- "cran-db.zip" + download.file(url, dest, method = "auto") +} + +extract_packages_with_versions <- function(osvs) { + result <- list() + + for (osv in osvs) { + for (affected in osv$affected) { + if (is.null(affected["package"]) || affected$package$ecosystem != "CRAN") { + next + } + + package <- affected$package$name + + if (!(package %in% names(result))) { + result[[package]] <- list() + } + + for (version in affected$versions) { + tryCatch( + { + as.package_version(version) + result[[package]] <- c(result[[package]], version) + }, + error = function(e) { + cat(sprintf("skipping invalid version %s for %s\n", version, package)) + } + ) + } + } + } + + # deduplicate and sort the versions for each package + for (package in names(result)) { + result[[package]] <- sort(numeric_version(unique(result[[package]]))) + } + + return(result) +} + +is_unsupported_comparison <- function(line) { + line %in% UNSUPPORTED_COMPARISONS +} + +uncomment <- function(line) { + if (startsWith(line, "#")) { + return(substr(line, 2, nchar(line))) + } + if (startsWith(line, "//")) { + return(substr(line, 3, nchar(line))) + } + return(line) +} + +compare <- function(v1, relate, v2) { + ops <- list('<' = function(result) result < 0, + '=' = function(result) result == 0, + '>' = function(result) result > 0) + + return(ops[[relate]](compareVersion(v1, v2))) +} + +compare_versions <- function(lines, select="all") { + has_any_failed <- FALSE + + for (line in lines) { + line <- trimws(line) + + if (line == "" || grepl("^#", line) || grepl("^//", line)) { + maybe_unsupported <- trimws(uncomment(line)) + + if (is_unsupported_comparison(maybe_unsupported)) { + cat(sprintf("\033[96mS\033[0m: \033[93m%s\033[0m\n", maybe_unsupported)) + } + next + } + + parts <- strsplit(trimws(line), " ")[[1]] + v1 <- parts[1] + op <- parts[2] + v2 <- parts[3] + + r <- compare(v1, op, v2) + + if (!r) { + has_any_failed <- TRUE + } + + if (select == "failures" && r) { + next + } + + if (select == "successes" && !r) { + next + } + + color <- ifelse(r, '\033[92m', '\033[91m') + rs <- ifelse(r, "T", "F") + cat(sprintf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line)) + } + return(has_any_failed) +} + +compare_versions_in_file <- function(filepath, select="all") { + lines <- readLines(filepath) + return(compare_versions(lines, select)) +} + +generate_version_compares <- function(versions) { + comparisons <- character() + + for (i in seq_along(versions)) { + if (i == 1) { + next + } + + comparison <- sprintf("%s < %s", versions[i - 1], versions[i]) + + if (is_unsupported_comparison(trimws(comparison))) { + comparison <- paste("#", comparison) + } + + comparisons <- c(comparisons, comparison) + } + + return(comparisons) +} + +generate_package_compares <- function(packages) { + comparisons <- character() + + for (package in names(packages)) { + versions <- packages[[package]] + comparisons <- c(comparisons, generate_version_compares(versions)) + } + + # return unique comparisons + return(unique(comparisons)) +} + +fetch_packages_versions <- function() { + download_cran_db() + osvs <- list() + + with_zip <- unzip("cran-db.zip", list = TRUE) + + for (fname in with_zip$Name) { + osv <- jsonlite::fromJSON(unzip("cran-db.zip", files = fname, exdir = tempdir()), simplifyDataFrame = FALSE) + osvs <- c(osvs, list(osv)) + } + + return(extract_packages_with_versions(osvs)) +} + +outfile <- "semantic/fixtures/cran-versions-generated.txt" + +packs <- fetch_packages_versions() +writeLines(generate_package_compares(packs), outfile, sep = "\n") +cat("\n") + +# set this to either "failures" or "successes" to only have those comparison results +# printed; setting it to anything else will have all comparison results printed +show <- Sys.getenv("VERSION_GENERATOR_PRINT", "failures") + +did_any_fail <- compare_versions_in_file(outfile, show) + +if (did_any_fail) { + q(status = 1) +} diff --git a/semantic/generators/generate-debian-versions.py b/semantic/generators/generate-debian-versions.py new file mode 100755 index 00000000..c3af8a3b --- /dev/null +++ b/semantic/generators/generate-debian-versions.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 + +import json +import operator +import os +import subprocess +import sys +import urllib.request +import zipfile +from pathlib import Path + +# this requires being run on an OS that has a version of "dpkg" which supports the +# "--compare-versions" option; also make sure to consider the version of dpkg being +# used in case there are changes to the comparing logic (last run with 1.19.7). +# +# also note that because of the large amount of versions being used there is +# significant overhead in having to use a subprocess, so this generator caches +# the results of said subprocess calls; a typical no-cache run takes about 5+ +# minutes whereas with the cache it only takes seconds. + +# An array of version comparisons that are known to be unsupported and so +# should be commented out in the generated fixture. +# +# Generally this is because the native implementation has a suspected bug +# that causes the comparison to return incorrect results, and so supporting +# such comparisons in the detector would in fact be wrong. +UNSUPPORTED_COMPARISONS = [] + + +def is_unsupported_comparison(line): + return line in UNSUPPORTED_COMPARISONS + + +def uncomment(line): + if line.startswith("#"): + return line[1:] + if line.startswith("//"): + return line[2:] + return line + + +def download_debian_db(): + urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip", "debian-db.zip") + + +def extract_packages_with_versions(osvs): + dict = {} + + for osv in osvs: + for affected in osv['affected']: + if 'package' not in affected or not affected['package']['ecosystem'].startswith('Debian'): + continue + + package = affected['package']['name'] + + if package not in dict: + dict[package] = [] + + for version in affected.get('versions', []): + dict[package].append(DebianVersion(version)) + + # deduplicate and sort the versions for each package + for package in dict: + dict[package] = sorted(list(dict.fromkeys(dict[package]))) + + return dict + + +class DebianVersionComparer: + def __init__(self, cache_path): + self.cache_path = Path(cache_path) + self.cache = {} + + self._load_cache() + + def _load_cache(self): + if self.cache_path: + self.cache_path.touch() + with open(self.cache_path, "r") as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + key, result = line.split(",") + + if result == "True": + self.cache[key] = True + continue + if result == "False": + self.cache[key] = False + continue + + print(f"ignoring invalid cache entry '{line}'") + + def _save_to_cache(self, key, result): + self.cache[key] = result + if self.cache_path: + self.cache_path.touch() + with open(self.cache_path, "a") as f: + f.write(f"{key},{result}\n") + + def compare(self, a, op, b): + key = f"{a} {op} {b}" + if key in self.cache: + return self.cache[key] + + cmd = ["dpkg", "--compare-versions", a, op, b] + out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if out.stdout: + print(out.stdout.decode('utf-8')) + if out.stderr: + print(out.stderr.decode('utf-8')) + + r = out.returncode == 0 + self._save_to_cache(key, r) + return r + + +debian_comparer = DebianVersionComparer("/tmp/debian-versions-generator-cache.csv") + + +class DebianVersion: + def __str__(self): + return self.version + + def __hash__(self): + return hash(self.version) + + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return debian_comparer.compare(self.version, 'lt', other.version) + + def __gt__(self, other): + return debian_comparer.compare(self.version, 'gt', other.version) + + def __eq__(self, other): + return debian_comparer.compare(self.version, 'eq', other.version) + + +def compare(v1, relate, v2): + ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt} + return ops[relate](v1, v2) + + +def compare_versions(lines, select="all"): + has_any_failed = False + + for line in lines: + line = line.strip() + + if line == "" or line.startswith('#') or line.startswith('//'): + maybe_unsupported = uncomment(line).strip() + + if is_unsupported_comparison(maybe_unsupported): + print(f"\033[96mS\033[0m: \033[93m{maybe_unsupported}\033[0m") + continue + + v1, op, v2 = line.strip().split(" ") + + r = compare(DebianVersion(v1), op, DebianVersion(v2)) + + if not r: + has_any_failed = r + + if select == "failures" and r: + continue + + if select == "successes" and not r: + continue + + color = '\033[92m' if r else '\033[91m' + rs = "T" if r else "F" + print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m") + return has_any_failed + + +def compare_versions_in_file(filepath, select="all"): + with open(filepath) as f: + lines = f.readlines() + return compare_versions(lines, select) + + +def generate_version_compares(versions): + comparisons = [] + for i, version in enumerate(versions): + if i == 0: + continue + + comparison = f"{versions[i - 1]} < {version}\n" + + if is_unsupported_comparison(comparison.strip()): + comparison = "# " + comparison + comparisons.append(comparison) + return comparisons + + +def generate_package_compares(packages): + comparisons = [] + for package in packages: + versions = packages[package] + comparisons.extend(generate_version_compares(versions)) + + # return comparisons + return list(dict.fromkeys(comparisons)) + + +def fetch_packages_versions(): + download_debian_db() + osvs = [] + + with zipfile.ZipFile('debian-db.zip') as db: + for fname in db.namelist(): + with db.open(fname) as osv: + osvs.append(json.loads(osv.read().decode('utf-8'))) + + return extract_packages_with_versions(osvs) + + +outfile = "semantic/fixtures/debian-versions-generated.txt" + +packs = fetch_packages_versions() +with open(outfile, "w") as f: + f.writelines(generate_package_compares(packs)) + f.write("\n") + +# set this to either "failures" or "successes" to only have those comparison results +# printed; setting it to anything else will have all comparison results printed +show = os.environ.get("VERSION_GENERATOR_PRINT", "failures") + +did_any_fail = compare_versions_in_file(outfile, show) + +if did_any_fail: + sys.exit(1) diff --git a/semantic/generators/generate-packagist-versions.php b/semantic/generators/generate-packagist-versions.php new file mode 100755 index 00000000..fbe03d84 --- /dev/null +++ b/semantic/generators/generate-packagist-versions.php @@ -0,0 +1,231 @@ +open($path, ZipArchive::RDONLY) === false) { + throw new RuntimeException('failed to read zip archive'); + } + + return $zip; +} + +/** + * @throws JsonException + * @throws RuntimeException + */ +function fetchPackageVersions(): array +{ + $dbPath = downloadPackagistDb(); + $dbZip = openDbZip($dbPath); + + $osvs = []; + + for ($i = 0; $i < $dbZip->numFiles; $i++) { + $file = $dbZip->getFromIndex($i); + + if ($file === false) { + throw new RuntimeException('failed to read a file from db zip'); + } + + $osvs[] = json_decode($file, true, 512, JSON_THROW_ON_ERROR); + } + + $packages = []; + + foreach ($osvs as $osv) { + foreach ($osv['affected'] as $affected) { + if (!isset($affected['package']) || $affected['package']['ecosystem'] !== 'Packagist') { + continue; + } + + $package = $affected['package']['name']; + + if (!isset($packages[$package])) { + $packages[$package] = []; + } + + if (empty($affected['versions'])) { + continue; + } + + foreach ($affected['versions'] as $version) { + $packages[$package][] = $version; + } + } + } + + return array_map(static function ($versions) { + $uniq = array_unique($versions); + usort($uniq, static fn($a, $b) => version_compare(ltrim($a, "vV"), ltrim($b, "vV"))); + + return $uniq; + }, $packages); +} + +/** + * Normalizes the previous version such that it will compare "correctly" to the current version, + * by ensuring that they both have the same "v" prefix (or lack of). + * + * Whether the "v" prefix is present on the normalized previous version depends on + * its presences in the current version; this ensure we will have _some_ versions that + * do have the "v" prefix, rather than it being present on _none_ or _all_ versions. + * + * @param string $currentVersion + * @param string $previousVersion + * + * @return string + */ +function normalizePrevVersion(string $currentVersion, string $previousVersion): string +{ + if (str_starts_with($currentVersion, "v")) { + $previousVersion = ltrim($previousVersion, "vV"); + + return "v$previousVersion"; + } + + if (str_starts_with($currentVersion, "V")) { + $previousVersion = ltrim($previousVersion, "vV"); + + return "V$previousVersion"; + } + + return ltrim($previousVersion, "vV"); +} + +function generateVersionCompares(array $versions): array +{ + $comparisons = []; + + foreach ($versions as $index => $version) { + if ($index === 0) { + continue; + } + + $prevVersion = normalizePrevVersion($version, $versions[$index - 1]); + $op = version_compare($prevVersion, $version) === 0 ? "=" : "<"; + + $comparison = "$prevVersion $op $version"; + + if (isUnsupportedComparison($comparison)) { + $comparison = "# $comparison"; + } + + $comparisons[] = $comparison; + } + + return $comparisons; +} + +function generatePackageCompares(array $packages): array +{ + $comparisons = []; + + foreach ($packages as $versions) { + $comparisons[] = generateVersionCompares($versions); + } + + return array_merge(...$comparisons); +} + +function compareVersions(array $lines, string $select = "all"): bool +{ + $hasAnyFailed = false; + + foreach ($lines as $line) { + $line = trim($line); + + if (empty($line) || str_starts_with($line, "#") || str_starts_with($line, "//")) { + $maybeUnsupported = trim(uncomment($line)); + + if (isUnsupportedComparison($maybeUnsupported)) { + echo "\033[96mS\033[0m: \033[93m$maybeUnsupported\033[0m\n"; + } + + continue; + } + + [$v1, $op, $v2] = explode(" ", $line); + + $r = version_compare($v1, $v2, $op); + + if (!$r) { + $hasAnyFailed = true; + } + + if ($select === "failures" && $r === true) { + continue; + } + + if ($select === "successes" && $r !== true) { + continue; + } + + $color = $r ? "\033[92m" : "\033[91m"; + $rs = $r ? "T" : "F"; + echo "$color$rs\033[0m: \033[93m$line\033[0m\n"; + } + + return $hasAnyFailed; +} + +$outfile = "semantic/fixtures/packagist-versions-generated.txt"; + +/** @noinspection PhpUnhandledExceptionInspection */ +$packages = fetchPackageVersions(); + +file_put_contents($outfile, implode("\n", array_unique(generatePackageCompares($packages))) . "\n"); + +// set this to either "failures" or "successes" to only have those comparison results +// printed; setting it to anything else will have all comparison results printed +$show = getenv("VERSION_GENERATOR_PRINT") ?: "failures"; + +$didAnyFail = compareVersions(explode("\n", file_get_contents($outfile)), $show); + +if ($didAnyFail === true) { + exit(1); +} diff --git a/semantic/generators/generate-pypi-versions.py b/semantic/generators/generate-pypi-versions.py new file mode 100755 index 00000000..f14a2d85 --- /dev/null +++ b/semantic/generators/generate-pypi-versions.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +import json +import operator +import os +import packaging.version +import sys +import urllib.request +import zipfile + +# this requires you run "pip install packaging" - have to be careful about versions too +# because of the "legacy version" stuff + +# An array of version comparisons that are known to be unsupported and so +# should be commented out in the generated fixture. +# +# Generally this is because the native implementation has a suspected bug +# that causes the comparison to return incorrect results, and so supporting +# such comparisons in the detector would in fact be wrong. +UNSUPPORTED_COMPARISONS = [] + + +def is_unsupported_comparison(line): + return line in UNSUPPORTED_COMPARISONS + + +def uncomment(line): + if line.startswith("#"): + return line[1:] + if line.startswith("//"): + return line[2:] + return line + + +def download_pypi_db(): + urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip", "pypi-db.zip") + + +def extract_packages_with_versions(osvs): + dict = {} + + for osv in osvs: + for affected in osv['affected']: + if 'package' not in affected or affected['package']['ecosystem'] != 'PyPI': + continue + + package = affected['package']['name'] + + if package not in dict: + dict[package] = [] + + for version in affected.get('versions', []): + try: + dict[package].append(packaging.version.parse(version)) + except packaging.version.InvalidVersion: + print(f"skipping invalid version {version} for {package}") + + # deduplicate and sort the versions for each package + for package in dict: + dict[package] = sorted(list(dict.fromkeys(dict[package]))) + + return dict + + +def compare(v1, relate, v2): + ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt} + return ops[relate](v1, v2) + + +def compare_versions(lines, select="all"): + has_any_failed = False + + for line in lines: + line = line.strip() + + if line == "" or line.startswith('#') or line.startswith('//'): + maybe_unsupported = uncomment(line).strip() + + if is_unsupported_comparison(maybe_unsupported): + print(f"\033[96mS\033[0m: \033[93m{maybe_unsupported}\033[0m") + continue + + v1, op, v2 = line.strip().split(" ") + + r = compare(packaging.version.parse(v1), op, packaging.version.parse(v2)) + + if not r: + has_any_failed = True + + if select == "failures" and r: + continue + + if select == "successes" and not r: + continue + + color = '\033[92m' if r else '\033[91m' + rs = "T" if r else "F" + print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m") + return has_any_failed + + +def compare_versions_in_file(filepath, select="all"): + with open(filepath) as f: + lines = f.readlines() + return compare_versions(lines, select) + + +def generate_version_compares(versions): + comparisons = [] + for i, version in enumerate(versions): + if i == 0: + continue + + comparison = f"{versions[i - 1]} < {version}\n" + + if is_unsupported_comparison(comparison.strip()): + comparison = "# " + comparison + comparisons.append(comparison) + return comparisons + + +def generate_package_compares(packages): + comparisons = [] + for package in packages: + versions = packages[package] + comparisons.extend(generate_version_compares(versions)) + + # return comparisons + return list(dict.fromkeys(comparisons)) + + +def fetch_packages_versions(): + download_pypi_db() + osvs = [] + + with zipfile.ZipFile('pypi-db.zip') as db: + for fname in db.namelist(): + with db.open(fname) as osv: + osvs.append(json.loads(osv.read().decode('utf-8'))) + + return extract_packages_with_versions(osvs) + + +outfile = "semantic/fixtures/pypi-versions-generated.txt" + +packs = fetch_packages_versions() +with open(outfile, "w") as f: + f.writelines(generate_package_compares(packs)) + f.write("\n") + +# set this to either "failures" or "successes" to only have those comparison results +# printed; setting it to anything else will have all comparison results printed +show = os.environ.get("VERSION_GENERATOR_PRINT", "failures") + +did_any_fail = compare_versions_in_file(outfile, show) + +if did_any_fail: + sys.exit(1) diff --git a/semantic/generators/generate-rubygems-versions.rb b/semantic/generators/generate-rubygems-versions.rb new file mode 100755 index 00000000..b638c88b --- /dev/null +++ b/semantic/generators/generate-rubygems-versions.rb @@ -0,0 +1,147 @@ +#!/usr/bin/env ruby + +require "rubygems/version" +require "open-uri" +require "json" +require "zip" + +# An array of version comparisons that are known to be unsupported and so +# should be commented out in the generated fixture. +# +# Generally this is because the native implementation has a suspected bug +# that causes the comparison to return incorrect results, and so supporting +# such comparisons in the detector would in fact be wrong. +# +# @type [Array] +UNSUPPORTED_COMPARISONS = [].freeze + +# @param [String] line +# @return [Boolean] +def is_unsupported_comparison?(line) + UNSUPPORTED_COMPARISONS.include? line +end + +# @param [String] line +# @return [String] +def uncomment(line) + line.sub(%r{^#|//}, "") +end + +def download_rubygems_db + URI.open("https://osv-vulnerabilities.storage.googleapis.com/RubyGems/all.zip") do |zip| + File.binwrite("rubygems-db.zip", zip.read) + end +end + +def extract_packages_with_versions(osvs) + packages = {} + + osvs.each do |osv| + osv["affected"].each do |affected| + next unless affected.dig("package", "ecosystem") == "RubyGems" + + package = affected["package"]["name"] + + packages[package] ||= [] + affected.fetch("versions", []).each do |version| + packages[package] << Gem::Version.new(version) + end + end + end + + packages.transform_values { |v| v.uniq.sort } +end + +def compare_version(v1, op, v2) + op = "==" if op == "=" + + Gem::Version.new(v1).method(op).call(Gem::Version.new(v2)) +end + +# @param [Array] lines +# @return [Boolean] +def compare_versions(lines, select = :all) + has_any_failed = false + + lines.each do |line| + line = line.strip + + if line.empty? || line.start_with?("#") || line.start_with?("//") + maybe_unsupported = uncomment(line).strip + + puts "\033[96mS\033[0m: \033[93m#{maybe_unsupported}\033[0m" if is_unsupported_comparison?(maybe_unsupported) + + next + end + + parts = line.split + v1 = parts[0] + op = parts[1] + v2 = parts[2] + + r = compare_version(v1, op, v2) + + has_any_failed = true unless r + + next if select == :failures && r == true + next if select == :successes && r != true + + color = r ? "\033[92m" : "\033[91m" + rs = r ? "T" : "F" + puts "#{color}#{rs}\033[0m: \033[93m#{line}\033[0m" + end + + has_any_failed +end + +def compare_versions_in_file(filepath, select = :all) + compare_versions(File.readlines(filepath), select) +end + +def generate_version_compares(versions) + comparisons = [] + + versions.each_with_index do |version, i| + next if i == 0 + + op = "<" + op = "=" if versions[i - 1] == version + + comparison = "#{versions[i - 1]} #{op} #{version}" + comparison = "# #{comparison}" if is_unsupported_comparison?(comparison) + + comparisons << comparison + end + + comparisons +end + +def generate_package_compares(packages) + comparisons = [] + + packages.each_value { |versions| comparisons.concat(generate_version_compares(versions)) } + + comparisons +end + +def fetch_packages_versions + download_rubygems_db + + osvs = Zip::File.open("rubygems-db.zip").map { |f| JSON.parse(f.get_input_stream.read) } + + extract_packages_with_versions(osvs) +end + +outfile = "semantic/fixtures/rubygems-versions-generated.txt" + +packs = fetch_packages_versions + +File.write(outfile, "#{generate_package_compares(packs).uniq.join("\n")}\n") + +# set this to either "failures" or "successes" to only have those comparison results +# printed; setting it to anything else will have all comparison results printed +show = ENV.fetch("VERSION_GENERATOR_PRINT", :failures).to_sym + +did_any_fail = compare_versions_in_file(outfile, show) + +exit(1) if did_any_fail