Merge pull request #226 from chmeliik/oci-copy-sbom-script

Add script for generating SBOM from oci-copy.yaml

.github/workflows/test-sbom-utility-scripts-image.yml

@@ -46,8 +46,14 @@ jobs:
         cd ./sbom-utility-scripts/scripts/index-image-sbom-script/
         tox
 
-    - name: Run tox checks for index-image-sbom-script
+    - name: Run tox checks for add-image-reference-script
       run: |
         python3 -m pip install tox
         cd ./sbom-utility-scripts/scripts/add-image-reference-script/
         tox
+
+    - name: Run tox checks for sbom-for-oci-copy-task
+      run: |
+        python3 -m pip install tox
+        cd ./sbom-utility-scripts/scripts/sbom-for-oci-copy-task/
+        tox

sbom-utility-scripts/Dockerfile

@@ -15,8 +15,12 @@ COPY scripts/index-image-sbom-script/index_image_sbom_script.py /scripts
 COPY scripts/add-image-reference-script/add_image_reference.py /scripts
 COPY scripts/add-image-reference-script/requirements.txt /scripts/add-image-reference-requirements.txt
 
+COPY scripts/sbom-for-oci-copy-task/sbom_for_oci_copy_task.py /scripts
+COPY scripts/sbom-for-oci-copy-task/requirements.txt /scripts/sbom-for-oci-copy-task-requirements.txt
+
 RUN pip3 install --no-cache-dir \
     -r merge-sboms-script-requirements.txt \
     -r base-images-sbom-script-requirements.txt \
     -r index-image-sbom-script-requirements.txt \
-    -r add-image-reference-requirements.txt
+    -r add-image-reference-requirements.txt \
+    -r sbom-for-oci-copy-task-requirements.txt

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/requirements-test.in

@@ -0,0 +1 @@
+pytest

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/requirements-test.txt

@@ -0,0 +1,22 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --generate-hashes --output-file=requirements-test.txt requirements-test.in
+#
+iniconfig==2.0.0 \
+    --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
+    --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
+    # via pytest
+packaging==24.1 \
+    --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
+    --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+    # via pytest
+pluggy==1.5.0 \
+    --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \
+    --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669
+    # via pytest
+pytest==8.3.2 \
+    --hash=sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5 \
+    --hash=sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce
+    # via -r requirements-test.in

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/requirements.in

@@ -0,0 +1,2 @@
+packageurl-python
+pyyaml

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/requirements.txt

@@ -0,0 +1,65 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile --generate-hashes requirements.in
+#
+packageurl-python==0.16.0 \
+    --hash=sha256:5c3872638b177b0f1cf01c3673017b7b27ebee485693ae12a8bed70fa7fa7c35 \
+    --hash=sha256:69e3bf8a3932fe9c2400f56aaeb9f86911ecee2f9398dbe1b58ec34340be365d
+    # via -r requirements.in
+pyyaml==6.0.2 \
+    --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
+    --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
+    --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \
+    --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \
+    --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \
+    --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \
+    --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \
+    --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \
+    --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \
+    --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \
+    --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \
+    --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \
+    --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \
+    --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \
+    --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \
+    --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \
+    --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \
+    --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \
+    --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \
+    --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \
+    --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \
+    --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \
+    --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \
+    --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \
+    --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \
+    --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \
+    --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \
+    --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \
+    --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \
+    --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \
+    --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \
+    --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \
+    --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \
+    --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \
+    --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \
+    --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \
+    --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \
+    --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \
+    --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \
+    --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \
+    --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \
+    --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \
+    --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \
+    --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \
+    --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \
+    --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \
+    --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \
+    --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \
+    --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \
+    --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \
+    --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \
+    --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \
+    --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4
+    # via -r requirements.in

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/sbom_for_oci_copy_task.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+import argparse
+import datetime
+import hashlib
+import json
+import re
+import uuid
+from typing import IO, Any, TypedDict
+
+import yaml
+from packageurl import PackageURL
+
+
+class Artifact(TypedDict):
+    # https://github.com/konflux-ci/build-definitions/blob/main/task/oci-copy/0.1/README.md#oci-copyyaml-schema
+    source: str
+    filename: str
+    type: str
+    sha256sum: str
+
+
+def to_purl(artifact: Artifact) -> str:
+    return PackageURL(
+        type="generic",
+        name=artifact["filename"],
+        qualifiers={
+            "download_url": artifact["source"],
+            "checksum": f"sha256:{artifact['sha256sum']}",
+        },
+    ).to_string()
+
+
+def to_cyclonedx_component(artifact: Artifact) -> dict[str, Any]:
+    return {
+        "type": "file",
+        "name": artifact["filename"],
+        "purl": to_purl(artifact),
+        "hashes": [{"alg": "SHA-256", "content": artifact["sha256sum"]}],
+        "externalReferences": [{"type": "distribution", "url": artifact["source"]}],
+    }
+
+
+def to_cyclonedx_sbom(artifacts: list[Artifact]) -> dict[str, Any]:
+    return {
+        "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
+        "bomFormat": "CycloneDX",
+        "specVersion": "1.5",
+        "version": 1,
+        "metadata": {},
+        "components": list(map(to_cyclonedx_component, artifacts)),
+    }
+
+
+def to_spdx_package(artifact: Artifact) -> dict[str, Any]:
+    purl = to_purl(artifact)
+    purl_hex_digest = hashlib.sha256(purl.encode()).hexdigest()
+    # based on a validation error from https://github.com/spdx/tools-java
+    #   Invalid SPDX ID: ...  Must match the pattern SPDXRef-([0-9a-zA-Z\.\-\+]+)$
+    sanitized_filename = re.sub(r"[^0-9a-zA-Z\.\-\+]", "-", artifact["filename"])
+    return {
+        "SPDXID": f"SPDXRef-Package-{sanitized_filename}-{purl_hex_digest}",
+        "name": artifact["filename"],
+        "externalRefs": [
+            {
+                "referenceType": "purl",
+                "referenceLocator": purl,
+                "referenceCategory": "PACKAGE-MANAGER",
+            },
+        ],
+        "checksums": [{"algorithm": "SHA256", "checksumValue": artifact["sha256sum"]}],
+        "downloadLocation": artifact["source"],
+    }
+
+
+def to_spdx_sbom(artifacts: list[Artifact]) -> dict[str, Any]:
+    real_packages = list(map(to_spdx_package, artifacts))
+
+    def relationship(a: str, relationship_type: str, b: str) -> dict[str, Any]:
+        return {"spdxElementId": a, "relationshipType": relationship_type, "relatedSpdxElement": b}
+
+    # The only purpose of this package is to be the "root" of the relationships graph
+    fake_root = {
+        "SPDXID": "SPDXRef-DocumentRoot-Unknown",
+        "downloadLocation": "NOASSERTION",
+        "name": "",
+    }
+
+    relationships = [relationship("SPDXRef-DOCUMENT", "DESCRIBES", fake_root["SPDXID"])]
+    relationships.extend(relationship(fake_root["SPDXID"], "CONTAINS", package["SPDXID"]) for package in real_packages)
+
+    packages = [fake_root] + real_packages
+
+    return {
+        "spdxVersion": "SPDX-2.3",
+        "dataLicense": "CC0-1.0",
+        "documentNamespace": f"https://konflux-ci.dev/spdxdocs/sbom-for-oci-copy-task/{uuid.uuid4()}",
+        "SPDXID": "SPDXRef-DOCUMENT",
+        "creationInfo": {
+            "created": _datetime_utc_now().strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "creators": ["Tool: Konflux"],
+        },
+        "name": "sbom-for-oci-copy-task",
+        "packages": packages,
+        "relationships": relationships,
+    }
+
+
+def _datetime_utc_now() -> datetime.datetime:
+    # a mockable datetime.datetime.now
+    return datetime.datetime.now(datetime.UTC)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("oci_copy_yaml", type=argparse.FileType(), default="-")
+    ap.add_argument("-o", "--output-file", type=argparse.FileType(mode="w"), default="-")
+    ap.add_argument("--sbom-type", choices=["cyclonedx", "spdx"], default="cyclonedx")
+    args = ap.parse_args()
+
+    oci_copy_yaml: IO[str] = args.oci_copy_yaml
+    output_file: IO[str] = args.output_file
+    sbom_type: str = args.sbom_type
+
+    oci_copy_data = yaml.safe_load(oci_copy_yaml)
+    artifacts: list[Artifact] = oci_copy_data["artifacts"]
+
+    if sbom_type == "cyclonedx":
+        sbom = to_cyclonedx_sbom(artifacts)
+    else:
+        sbom = to_spdx_sbom(artifacts)
+
+    json.dump(sbom, output_file, indent=2)
+    output_file.write("\n")
+
+
+if __name__ == "__main__":
+    main()

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/test_data/cyclonedx.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
+  "bomFormat": "CycloneDX",
+  "specVersion": "1.5",
+  "version": 1,
+  "metadata": {},
+  "components": [
+    {
+      "type": "file",
+      "name": "merlinite-7b-lab-Q4_K_M.gguf",
+      "purl": "pkg:generic/merlinite-7b-lab-Q4_K_M.gguf?checksum=sha256:9ca044d727db34750e1aeb04e3b18c3cf4a8c064a9ac96cf00448c506631d16c&download_url=https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/4bb27da133fc4888d687ab731ac7faf0ed804c6d/merlinite-7b-lab-Q4_K_M.gguf",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "9ca044d727db34750e1aeb04e3b18c3cf4a8c064a9ac96cf00448c506631d16c"
+        }
+      ],
+      "externalReferences": [
+        {
+          "type": "distribution",
+          "url": "https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/4bb27da133fc4888d687ab731ac7faf0ed804c6d/merlinite-7b-lab-Q4_K_M.gguf"
+        }
+      ]
+    },
+    {
+      "type": "file",
+      "name": "huggingface.svg",
+      "purl": "pkg:generic/huggingface.svg?checksum=sha256:3613c73f07ccae19118bfe6d2f8cd127183d08cf99468a708e090953e116ed0a&download_url=https://huggingface.co/front/assets/huggingface_logo-noborder.svg",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "3613c73f07ccae19118bfe6d2f8cd127183d08cf99468a708e090953e116ed0a"
+        }
+      ],
+      "externalReferences": [
+        {
+          "type": "distribution",
+          "url": "https://huggingface.co/front/assets/huggingface_logo-noborder.svg"
+        }
+      ]
+    }
+  ]
+}

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/test_data/oci-copy.yaml

@@ -0,0 +1,11 @@
+# from https://github.com/ralphbean/merlinite-poc/blob/main/oci-copy.yaml
+artifact_type: application/x-mlmodel
+artifacts:
+  - source: https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/4bb27da133fc4888d687ab731ac7faf0ed804c6d/merlinite-7b-lab-Q4_K_M.gguf
+    filename: merlinite-7b-lab-Q4_K_M.gguf
+    type: application/vnd.gguf
+    sha256sum: 9ca044d727db34750e1aeb04e3b18c3cf4a8c064a9ac96cf00448c506631d16c
+  - source: https://huggingface.co/front/assets/huggingface_logo-noborder.svg
+    filename: huggingface.svg
+    type: image/svg+xml
+    sha256sum: 3613c73f07ccae19118bfe6d2f8cd127183d08cf99468a708e090953e116ed0a

sbom-utility-scripts/scripts/sbom-for-oci-copy-task/test_data/spdx.json

@@ -0,0 +1,73 @@
+{
+  "spdxVersion": "SPDX-2.3",
+  "dataLicense": "CC0-1.0",
+  "documentNamespace": "https://konflux-ci.dev/spdxdocs/sbom-for-oci-copy-task/a29a127a-daf6-44d3-a840-4eca194e9b41",
+  "SPDXID": "SPDXRef-DOCUMENT",
+  "creationInfo": {
+    "created": "2025-01-14T11:46:34Z",
+    "creators": [
+      "Tool: Konflux"
+    ]
+  },
+  "name": "sbom-for-oci-copy-task",
+  "packages": [
+    {
+      "SPDXID": "SPDXRef-DocumentRoot-Unknown",
+      "downloadLocation": "NOASSERTION",
+      "name": ""
+    },
+    {
+      "SPDXID": "SPDXRef-Package-merlinite-7b-lab-Q4-K-M.gguf-8809169785e5ac29bd1777171256c0c4f4b584dbc5838bf9178ac72c1dc23585",
+      "name": "merlinite-7b-lab-Q4_K_M.gguf",
+      "externalRefs": [
+        {
+          "referenceType": "purl",
+          "referenceLocator": "pkg:generic/merlinite-7b-lab-Q4_K_M.gguf?checksum=sha256:9ca044d727db34750e1aeb04e3b18c3cf4a8c064a9ac96cf00448c506631d16c&download_url=https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/4bb27da133fc4888d687ab731ac7faf0ed804c6d/merlinite-7b-lab-Q4_K_M.gguf",
+          "referenceCategory": "PACKAGE-MANAGER"
+        }
+      ],
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "9ca044d727db34750e1aeb04e3b18c3cf4a8c064a9ac96cf00448c506631d16c"
+        }
+      ],
+      "downloadLocation": "https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/4bb27da133fc4888d687ab731ac7faf0ed804c6d/merlinite-7b-lab-Q4_K_M.gguf"
+    },
+    {
+      "SPDXID": "SPDXRef-Package-huggingface.svg-7206c6f7c832ae92d3c1e09864c981221c4371209d05e13862d5d93eaafc7c04",
+      "name": "huggingface.svg",
+      "externalRefs": [
+        {
+          "referenceType": "purl",
+          "referenceLocator": "pkg:generic/huggingface.svg?checksum=sha256:3613c73f07ccae19118bfe6d2f8cd127183d08cf99468a708e090953e116ed0a&download_url=https://huggingface.co/front/assets/huggingface_logo-noborder.svg",
+          "referenceCategory": "PACKAGE-MANAGER"
+        }
+      ],
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "3613c73f07ccae19118bfe6d2f8cd127183d08cf99468a708e090953e116ed0a"
+        }
+      ],
+      "downloadLocation": "https://huggingface.co/front/assets/huggingface_logo-noborder.svg"
+    }
+  ],
+  "relationships": [
+    {
+      "spdxElementId": "SPDXRef-DOCUMENT",
+      "relationshipType": "DESCRIBES",
+      "relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown"
+    },
+    {
+      "spdxElementId": "SPDXRef-DocumentRoot-Unknown",
+      "relationshipType": "CONTAINS",
+      "relatedSpdxElement": "SPDXRef-Package-merlinite-7b-lab-Q4-K-M.gguf-8809169785e5ac29bd1777171256c0c4f4b584dbc5838bf9178ac72c1dc23585"
+    },
+    {
+      "spdxElementId": "SPDXRef-DocumentRoot-Unknown",
+      "relationshipType": "CONTAINS",
+      "relatedSpdxElement": "SPDXRef-Package-huggingface.svg-7206c6f7c832ae92d3c1e09864c981221c4371209d05e13862d5d93eaafc7c04"
+    }
+  ]
+}