https://github.com/python/cpython/commit/72dae53e09a5344bf4922d934a34a2fa48a... commit: 72dae53e09a5344bf4922d934a34a2fa48a11c86 branch: main author: Seth Michael Larson <seth@python.org> committer: zooba <steve.dower@microsoft.com> date: 2024-04-30T16:05:05+01:00 summary: gh-116122: Add SBOM generation to PCbuild/build.bat (GH-116138) files: M PCbuild/regen.targets M Tools/build/generate_sbom.py diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index b72ef5b5c6055c..4aa14ed1fad9eb 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -33,11 +33,15 @@ <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> + <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json"> + <Format>json</Format> + </_SbomOutputs> </ItemGroup> <Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'"> <Message Text="Touching source files to force regeneration" Importance="high" /> - <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs);@(_CasesSources)" + <Touch Files="@(_PegenSources);@(_ASTSources);@(_TokenSources);@(_KeywordOutputs);@(_CasesSources);@(_SbomSources)" AlwaysCreate="False" /> </Target> @@ -126,7 +130,14 @@ DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> </Target> - <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT;_RegenCases"> + <Target Name="_RegenSbom" + DependsOnTargets="FindPythonForBuild" + Inputs="@(_SbomSources)" + Outputs="@(_SbomOutputs)"> + <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\generate_sbom.py"'/> + </Target> + + <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT;_RegenCases;_RegenSbom"> <Message Text="Generated sources are up to date" Importance="high" /> </Target> diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 5c1851f09338a0..258b58c03c6800 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,13 +4,13 @@ import hashlib import json import glob -import pathlib +from pathlib import Path, PurePosixPath, PureWindowsPath import subprocess import sys import urllib.request import typing -CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent +CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent # Before adding a new entry to this list, double check that # the license expression is a valid SPDX license expression: @@ -119,9 +119,16 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: # 1 means matches, 0 means no matches. assert git_check_ignore_proc.returncode in (0, 1) + # Paths may or may not be quoted, Windows quotes paths. + git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z") + # Return the list of paths sorted git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines() - return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) + git_check_not_ignored = [] + for line in git_check_ignore_lines: + if match := git_check_ignore_re.fullmatch(line): + git_check_not_ignored.append(match.group(2) or match.group(3)) + return sorted(git_check_not_ignored) def get_externals() -> list[str]: @@ -238,12 +245,20 @@ def create_source_sbom() -> None: ) for path in paths: + + # Normalize the filename from any combination of slashes. + path = str(PurePosixPath(PureWindowsPath(path))) + # Skip directories and excluded files if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude: continue # SPDX requires SHA1 to be used for files, but we provide SHA256 too. data = (CPYTHON_ROOT_DIR / path).read_bytes() + # We normalize line-endings for consistent checksums. + # This is a rudimentary check for binary files. + if b"\x00" not in data: + data = data.replace(b"\r\n", b"\n") checksum_sha1 = hashlib.sha1(data).hexdigest() checksum_sha256 = hashlib.sha256(data).hexdigest()