gh-116122: Add SBOM generation to PCbuild/build.bat (GH-116138)

Author: sethmlarson

PCbuild/regen.targets

@@ -33,11 +33,15 @@
     <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
     <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/>
     <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/>
+    <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" />
+    <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json">
+      <Format>json</Format>
+    </_SbomOutputs>
   </ItemGroup>
 
   <Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'">
     <Message Text="Touching source files to force regeneration" Importance="high" />
-    <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs);@(_CasesSources)"
+    <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs);@(_CasesSources);@(_SbomSources)"
            AlwaysCreate="False" />
   </Target>
 
@@ -126,7 +130,14 @@
           DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects">
   </Target>
 
-  <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT;_RegenCases">
+  <Target Name="_RegenSbom"
+          DependsOnTargets="FindPythonForBuild"
+          Inputs="@(_SbomSources)"
+          Outputs="@(_SbomOutputs)">
+    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\generate_sbom.py"'/>
+  </Target>
+
+  <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT;_RegenCases;_RegenSbom">
     <Message Text="Generated sources are up to date" Importance="high" />
   </Target>
 

Tools/build/generate_sbom.py

@@ -4,13 +4,13 @@
 import hashlib
 import json
 import glob
-import pathlib
+from pathlib import Path, PurePosixPath, PureWindowsPath
 import subprocess
 import sys
 import urllib.request
 import typing
 
-CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent
+CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent
 
 # Before adding a new entry to this list, double check that
 # the license expression is a valid SPDX license expression:
@@ -119,9 +119,16 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]:
     # 1 means matches, 0 means no matches.
     assert git_check_ignore_proc.returncode in (0, 1)
 
+    # Paths may or may not be quoted, Windows quotes paths.
+    git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z")
+
     # Return the list of paths sorted
     git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines()
-    return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")])
+    git_check_not_ignored = []
+    for line in git_check_ignore_lines:
+        if match := git_check_ignore_re.fullmatch(line):
+            git_check_not_ignored.append(match.group(2) or match.group(3))
+    return sorted(git_check_not_ignored)
 
 
 def get_externals() -> list[str]:
@@ -238,12 +245,20 @@ def create_source_sbom() -> None:
             )
 
             for path in paths:
+
+                # Normalize the filename from any combination of slashes.
+                path = str(PurePosixPath(PureWindowsPath(path)))
+
                 # Skip directories and excluded files
                 if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude:
                     continue
 
                 # SPDX requires SHA1 to be used for files, but we provide SHA256 too.
                 data = (CPYTHON_ROOT_DIR / path).read_bytes()
+                # We normalize line-endings for consistent checksums.
+                # This is a rudimentary check for binary files.
+                if b"\x00" not in data:
+                    data = data.replace(b"\r\n", b"\n")
                 checksum_sha1 = hashlib.sha1(data).hexdigest()
                 checksum_sha256 = hashlib.sha256(data).hexdigest()