diff --git a/src/macaron/build_spec_generator/build_command_patcher.py b/src/macaron/build_spec_generator/build_command_patcher.py index 489cf8fb9..4fe26f2ba 100644 --- a/src/macaron/build_spec_generator/build_command_patcher.py +++ b/src/macaron/build_spec_generator/build_command_patcher.py @@ -26,6 +26,45 @@ PatchValueType = GradleOptionPatchValueType | MavenOptionPatchValueType +CLI_COMMAND_PATCHES: dict[ + PatchCommandBuildTool, + Mapping[str, PatchValueType | None], +] = { + PatchCommandBuildTool.MAVEN: { + "goals": ["clean", "package"], + "--batch-mode": False, + "--quiet": False, + "--no-transfer-progress": False, + # Example pkg:maven/io.liftwizard/liftwizard-servlet-logging-mdc@1.0.1 + # https://github.com/liftwizard/liftwizard/blob/ + # 4ea841ffc9335b22a28a7a19f9156e8ba5820027/.github/workflows/build-and-test.yml#L23 + "--threads": None, + # For cases such as + # pkg:maven/org.apache.isis.valuetypes/isis-valuetypes-prism-resources@2.0.0-M7 + "--version": False, + "--define": { + # pkg:maven/org.owasp/dependency-check-utils@7.3.2 + # To remove "-Dgpg.passphrase=$MACARON_UNKNOWN" + "gpg.passphrase": None, + "skipTests": "true", + "maven.test.skip": "true", + "maven.site.skip": "true", + "rat.skip": "true", + "maven.javadoc.skip": "true", + }, + }, + PatchCommandBuildTool.GRADLE: { + "tasks": ["clean", "assemble"], + "--console": "plain", + "--exclude-task": ["test"], + "--project-prop": { + "skip.signing": "", + "skipSigning": "", + "gnupg.skip": "", + }, + }, +} + def _patch_commands( cmds_sequence: Sequence[list[str]], diff --git a/src/macaron/build_spec_generator/build_spec_generator.py b/src/macaron/build_spec_generator/build_spec_generator.py index ac89ce2fd..dfa6e9e24 100644 --- a/src/macaron/build_spec_generator/build_spec_generator.py +++ b/src/macaron/build_spec_generator/build_spec_generator.py @@ -6,14 +6,12 @@ import json import logging import os -from collections.abc import Mapping from enum import Enum from packageurl import PackageURL from sqlalchemy import create_engine from sqlalchemy.orm import Session -from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType from macaron.build_spec_generator.common_spec.core import gen_generic_build_spec from macaron.build_spec_generator.reproducible_central.reproducible_central import gen_reproducible_central_build_spec from macaron.console import access_handler @@ -31,46 +29,6 @@ class BuildSpecFormat(str, Enum): DEFAULT = "default-buildspec" -CLI_COMMAND_PATCHES: dict[ - PatchCommandBuildTool, - Mapping[str, PatchValueType | None], -] = { - PatchCommandBuildTool.MAVEN: { - "goals": ["clean", "package"], - "--batch-mode": False, - "--quiet": False, - "--no-transfer-progress": False, - # Example pkg:maven/io.liftwizard/liftwizard-servlet-logging-mdc@1.0.1 - # https://github.com/liftwizard/liftwizard/blob/ - # 4ea841ffc9335b22a28a7a19f9156e8ba5820027/.github/workflows/build-and-test.yml#L23 - "--threads": None, - # For cases such as - # pkg:maven/org.apache.isis.valuetypes/isis-valuetypes-prism-resources@2.0.0-M7 - "--version": False, - "--define": { - # pkg:maven/org.owasp/dependency-check-utils@7.3.2 - # To remove "-Dgpg.passphrase=$MACARON_UNKNOWN" - "gpg.passphrase": None, - "skipTests": "true", - "maven.test.skip": "true", - "maven.site.skip": "true", - "rat.skip": "true", - "maven.javadoc.skip": "true", - }, - }, - PatchCommandBuildTool.GRADLE: { - "tasks": ["clean", "assemble"], - "--console": "plain", - "--exclude-task": ["test"], - "--project-prop": { - "skip.signing": "", - "skipSigning": "", - "gnupg.skip": "", - }, - }, -} - - def gen_build_spec_for_purl( purl: PackageURL, database_path: str, @@ -113,7 +71,7 @@ def gen_build_spec_for_purl( with Session(db_engine) as session, session.begin(): try: - build_spec = gen_generic_build_spec(purl=purl, session=session, patches=CLI_COMMAND_PATCHES) + build_spec = gen_generic_build_spec(purl=purl, session=session) except GenerateBuildSpecError as error: logger.error("Error while generating the build spec: %s.", error) return os.EX_DATAERR @@ -128,7 +86,7 @@ def gen_build_spec_for_purl( # Default build spec. case BuildSpecFormat.DEFAULT: try: - build_spec_content = json.dumps(build_spec) + build_spec_content = json.dumps(build_spec, indent=4) except ValueError as error: logger.error("Error while serializing the build spec: %s.", error) return os.EX_DATAERR diff --git a/src/macaron/build_spec_generator/common_spec/base_spec.py b/src/macaron/build_spec_generator/common_spec/base_spec.py index 11b5e8850..952412983 100644 --- a/src/macaron/build_spec_generator/common_spec/base_spec.py +++ b/src/macaron/build_spec_generator/common_spec/base_spec.py @@ -92,3 +92,26 @@ def resolve_fields(self, purl: PackageURL) -> None: This method should be implemented by subclasses to handle logic specific to a given package ecosystem, such as Maven or PyPI. """ + + @abstractmethod + def get_default_build_command( + self, + build_tool_name: str, + ) -> list[str]: + """Return a default build command for the build tool. + + Parameters + ---------- + build_tool_name: str + The build tool to get the default build command. + + Returns + ------- + list[str] + The build command as a list[str]. + + Raises + ------ + GenerateBuildSpecError + If there is no default build command available for the specified build tool. + """ diff --git a/src/macaron/build_spec_generator/common_spec/core.py b/src/macaron/build_spec_generator/common_spec/core.py index d13ff3d2c..94dd8985d 100644 --- a/src/macaron/build_spec_generator/common_spec/core.py +++ b/src/macaron/build_spec_generator/common_spec/core.py @@ -6,15 +6,13 @@ import logging import pprint import shlex -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from enum import Enum from importlib import metadata as importlib_metadata -from pprint import pformat import sqlalchemy.orm from packageurl import PackageURL -from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType, patch_commands from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict from macaron.build_spec_generator.common_spec.maven_spec import MavenBuildSpec from macaron.build_spec_generator.common_spec.pypi_spec import PyPIBuildSpec @@ -119,45 +117,6 @@ def compose_shell_commands(cmds_sequence: list[list[str]]) -> str: return result -def get_default_build_command( - build_tool_name: MacaronBuildToolName, -) -> list[str] | None: - """Return a default build command for the build tool. - - Parameters - ---------- - build_tool_name: MacaronBuildToolName - The type of build tool to get the default build command. - - Returns - ------- - list[str] | None - The build command as a list[str] or None if we cannot get one for this tool. - """ - default_build_command = None - - match build_tool_name: - case MacaronBuildToolName.MAVEN: - default_build_command = "mvn clean package".split() - case MacaronBuildToolName.GRADLE: - default_build_command = "./gradlew clean assemble publishToMavenLocal".split() - case MacaronBuildToolName.PIP: - default_build_command = "python -m build".split() - case MacaronBuildToolName.POETRY: - default_build_command = "poetry build".split() - case _: - pass - - if not default_build_command: - logger.critical( - "There is no default build command available for the build tool %s.", - build_tool_name, - ) - return None - - return default_build_command - - def get_macaron_build_tool_name( build_tool_facts: Sequence[BuildToolFacts], target_language: str ) -> MacaronBuildToolName | None: @@ -322,10 +281,6 @@ def get_language_version( def gen_generic_build_spec( purl: PackageURL, session: sqlalchemy.orm.Session, - patches: Mapping[ - PatchCommandBuildTool, - Mapping[str, PatchValueType | None], - ], ) -> BaseBuildSpecDict: """ Generate and return the Buildspec file. @@ -336,9 +291,6 @@ def gen_generic_build_spec( The PackageURL to generate build spec for. session : sqlalchemy.orm.Session The SQLAlchemy Session opened for the database to extract build information. - patches : Mapping[PatchCommandBuildTool, Mapping[str, PatchValueType | None]] - The patches to apply to the build commands in ``build_info`` before being populated in - the output Buildspec. Returns ------- @@ -360,12 +312,6 @@ def gen_generic_build_spec( f"PURL type '{purl.type}' is not supported. Supported: {[e.name.lower() for e in ECOSYSTEMS]}" ) - logger.debug( - "Generating build spec for %s with command patches:\n%s", - purl, - pformat(patches), - ) - target_language = LANGUAGES[purl.type.upper()].value group = purl.namespace artifact = purl.name @@ -414,22 +360,7 @@ def gen_generic_build_spec( build_command_info or "Cannot find any.", ) - selected_build_command = ( - build_command_info.command - if build_command_info - else get_default_build_command( - build_tool_name, - ) - ) - if not selected_build_command: - raise GenerateBuildSpecError(f"Failed to get a build command for {purl}.") - - patched_build_commands = patch_commands( - cmds_sequence=[selected_build_command], - patches=patches, - ) - if not patched_build_commands: - raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_command}.") + selected_build_command = build_command_info.command if build_command_info else [] lang_version = get_language_version(build_command_info) if build_command_info else "" @@ -447,7 +378,7 @@ def gen_generic_build_spec( "purl": str(purl), "language": target_language, "build_tool": build_tool_name.value, - "build_commands": patched_build_commands, + "build_commands": [selected_build_command], } ) ECOSYSTEMS[purl.type.upper()].value(base_build_spec_dict).resolve_fields(purl) diff --git a/src/macaron/build_spec_generator/common_spec/maven_spec.py b/src/macaron/build_spec_generator/common_spec/maven_spec.py index ddfe96b71..f893896bc 100644 --- a/src/macaron/build_spec_generator/common_spec/maven_spec.py +++ b/src/macaron/build_spec_generator/common_spec/maven_spec.py @@ -8,9 +8,11 @@ from packageurl import PackageURL +from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict from macaron.build_spec_generator.common_spec.jdk_finder import find_jdk_version_from_central_maven_repo from macaron.build_spec_generator.common_spec.jdk_version_normalizer import normalize_jdk_version +from macaron.errors import GenerateBuildSpecError logger: logging.Logger = logging.getLogger(__name__) @@ -29,6 +31,46 @@ def __init__(self, data: BaseBuildSpecDict): """ self.data = data + def get_default_build_command( + self, + build_tool_name: str, + ) -> list[str]: + """Return a default build command for the build tool. + + Parameters + ---------- + build_tool_name: str + The build tool to get the default build command. + + Returns + ------- + list[str] + The build command as a list[str]. + + Raises + ------ + GenerateBuildSpecError + If there is no default build command available for the specified build tool. + """ + default_build_command = None + + match build_tool_name: + case "maven": + default_build_command = "mvn clean package".split() + case "gradle": + default_build_command = "./gradlew clean assemble publishToMavenLocal".split() + case _: + pass + + if not default_build_command: + logger.critical( + "There is no default build command available for the build tool %s.", + build_tool_name, + ) + raise GenerateBuildSpecError("Unable to find a default build command.") + + return default_build_command + def resolve_fields(self, purl: PackageURL) -> None: """ Resolve Maven-specific fields in the build specification. @@ -69,3 +111,17 @@ def resolve_fields(self, purl: PackageURL) -> None: return self.data["language_version"] = [major_jdk_version] + + # Resolve and patch build commands. + selected_build_commands = self.data["build_commands"] or [ + self.get_default_build_command(self.data["build_tool"]) + ] + + patched_build_commands = patch_commands( + cmds_sequence=selected_build_commands, + patches=CLI_COMMAND_PATCHES, + ) + if not patched_build_commands: + raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.") + + self.data["build_commands"] = patched_build_commands diff --git a/src/macaron/build_spec_generator/common_spec/pypi_spec.py b/src/macaron/build_spec_generator/common_spec/pypi_spec.py index 9c95e3dab..ee8d3da7c 100644 --- a/src/macaron/build_spec_generator/common_spec/pypi_spec.py +++ b/src/macaron/build_spec_generator/common_spec/pypi_spec.py @@ -12,9 +12,11 @@ from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import InvalidWheelFilename, parse_wheel_filename +from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict from macaron.config.defaults import defaults -from macaron.errors import SourceCodeError +from macaron.errors import GenerateBuildSpecError, SourceCodeError +from macaron.json_tools import json_extract from macaron.slsa_analyzer.package_registry import pypi_registry from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo @@ -37,6 +39,46 @@ def __init__(self, data: BaseBuildSpecDict): """ self.data = data + def get_default_build_command( + self, + build_tool_name: str, + ) -> list[str]: + """Return a default build command for the build tool. + + Parameters + ---------- + build_tool_name: str + The build tool to get the default build command. + + Returns + ------- + list[str] + The build command as a list[str]. + + Raises + ------ + GenerateBuildSpecError + If there is no default build command available for the specified build tool. + """ + default_build_command = None + + match build_tool_name: + case "pip": + default_build_command = "python -m build".split() + case "poetry": + default_build_command = "poetry build".split() + case _: + pass + + if not default_build_command: + logger.critical( + "There is no default build command available for the build tool %s.", + build_tool_name, + ) + raise GenerateBuildSpecError("Unable to find a default build command.") + + return default_build_command + def resolve_fields(self, purl: PackageURL) -> None: """ Resolve PyPI-specific fields in the build specification. @@ -46,7 +88,7 @@ def resolve_fields(self, purl: PackageURL) -> None: purl: str The target software component Package URL. """ - if purl.type != "pypi": + if purl.type != "pypi" or purl.version is None: return registry = pypi_registry.PyPIRegistry() @@ -60,23 +102,35 @@ def resolve_fields(self, purl: PackageURL) -> None: ) pypi_package_json = pypi_registry.find_or_create_pypi_asset(purl.name, purl.version, registry_info) + patched_build_commands: list[list[str]] = [] if pypi_package_json is not None: if pypi_package_json.package_json or pypi_package_json.download(dest=""): requires_array: list[str] = [] build_backends: dict[str, str] = {} - python_version_list: list[str] = [] + python_version_set: set[str] = set() + wheel_name_python_version_list: list[str] = [] + wheel_name_platforms: set[str] = set() + + # Get the Python constraints from the PyPI JSON response. + json_releases = pypi_package_json.get_releases() + if json_releases: + releases = json_extract(json_releases, [purl.version], list) or [] + for release in releases: + if py_version := json_extract(release, ["requires_python"], str): + python_version_set.add(py_version.replace(" ", "")) + try: with pypi_package_json.wheel(): logger.debug("Wheel at %s", pypi_package_json.wheel_path) - # Should only have .dist-info directory + # Should only have .dist-info directory. logger.debug("It has directories %s", ",".join(os.listdir(pypi_package_json.wheel_path))) wheel_contents, metadata_contents = self.read_directory(pypi_package_json.wheel_path, purl) generator, version = self.read_generator_line(wheel_contents) if generator != "": build_backends[generator] = "==" + version if generator != "setuptools": - # Apply METADATA heuristics to determine setuptools version + # Apply METADATA heuristics to determine setuptools version. if "License-File" in metadata_contents: build_backends["setuptools"] = "==" + defaults.get( "heuristic.pypi", "setuptools_version_emitting_license" @@ -102,9 +156,10 @@ def resolve_fields(self, purl: PackageURL) -> None: content = tomli.loads(pyproject_content.decode("utf-8")) build_system: dict[str, list[str]] = content.get("build-system", {}) requires_array = build_system.get("requires", []) - python_version_constraint = content.get("project", {}).get("requires-python") + + python_version_constraint = json_extract(content, ["project", "requires-python"], str) if python_version_constraint: - python_version_list.append(python_version_constraint) + python_version_set.add(python_version_constraint.replace(" ", "")) logger.debug("From pyproject.toml:") logger.debug(requires_array) except SourceCodeError: @@ -127,18 +182,39 @@ def resolve_fields(self, purl: PackageURL) -> None: logger.debug(build_backends) self.data["build_backends"] = build_backends - if not python_version_list: - try: - # Get python version specified in the wheel file name - logger.debug(pypi_package_json.wheel_filename) - _, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename) - for tag in tags: - python_version_list.append(tag.interpreter) - logger.debug(python_version_list) - except InvalidWheelFilename: - logger.debug("Could not parse wheel file name to extract version") - - self.data["language_version"] = python_version_list + try: + # Get information from the wheel file name. + logger.debug(pypi_package_json.wheel_filename) + _, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename) + for tag in tags: + wheel_name_python_version_list.append(tag.interpreter) + wheel_name_platforms.add(tag.platform) + logger.debug(python_version_set) + except InvalidWheelFilename: + logger.debug("Could not parse wheel file name to extract version") + + self.data["language_version"] = list(python_version_set) or wheel_name_python_version_list + + # Use the default build command for pure Python packages. + if "any" in wheel_name_platforms: + patched_build_commands = [self.get_default_build_command(self.data["build_tool"])] + + if not patched_build_commands: + # Resolve and patch build commands. + selected_build_commands = self.data["build_commands"] or [ + self.get_default_build_command(self.data["build_tool"]) + ] + patched_build_commands = ( + patch_commands( + cmds_sequence=selected_build_commands, + patches=CLI_COMMAND_PATCHES, + ) + or [] + ) + if not patched_build_commands: + raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.") + + self.data["build_commands"] = patched_build_commands def read_directory(self, wheel_path: str, purl: PackageURL) -> tuple[str, str]: """ diff --git a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec index b7ef0f639..f1622f4e7 100644 --- a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec +++ b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec @@ -1,5 +1,5 @@ -# Generated by Macaron version 0.17.0 +# Generated by Macaron version 0.18.0 groupId=io.github.behnazh-w.demo artifactId=core diff --git a/tests/integration/cases/micronaut-projects_micronaut-core/expected_reproducible_central.buildspec b/tests/integration/cases/micronaut-projects_micronaut-core/expected_reproducible_central.buildspec index 8caca83d6..a8d436d88 100644 --- a/tests/integration/cases/micronaut-projects_micronaut-core/expected_reproducible_central.buildspec +++ b/tests/integration/cases/micronaut-projects_micronaut-core/expected_reproducible_central.buildspec @@ -1,11 +1,5 @@ -# Copyright (c) 2025, Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -# Generated by Macaron version 0.15.0 - -# Input PURL - pkg:maven/io.micronaut/micronaut-core@4.2.3 -# Initial default JDK version 8 and default build command [['./gradlew', '-x', 'test', '-Pskip.signing', '-PskipSigning', '-Pgnupg.skip', 'clean', 'assemble']]. -# The lookup build command: ['./gradlew', 'publishToSonatype', 'closeAndReleaseSonatypeStagingRepository'] -# Jdk version from lookup build command 17. + +# Generated by Macaron version 0.18.0 groupId=io.micronaut artifactId=micronaut-core diff --git a/tests/integration/cases/pypi_toga/expected_default.buildspec b/tests/integration/cases/pypi_toga/expected_default.buildspec index d5335b3e5..2418f3909 100644 --- a/tests/integration/cases/pypi_toga/expected_default.buildspec +++ b/tests/integration/cases/pypi_toga/expected_default.buildspec @@ -1 +1,28 @@ -{"macaron_version": "0.18.0", "group_id": null, "artifact_id": "toga", "version": "0.5.1", "git_repo": "https://github.com/beeware/toga", "git_tag": "ef1912b0a1b5c07793f9aa372409f5b9d36f2604", "newline": "lf", "language_version": [">= 3.9"], "ecosystem": "pypi", "purl": "pkg:pypi/toga@0.5.1", "language": "python", "build_tool": "pip", "build_commands": [["pip", "install", "-U", "pip"]], "build_backends": {"setuptools": "==(80.3.1)", "setuptools_scm": "==8.3.1", "setuptools_dynamic_dependencies": "==1.0.0"}} +{ + "macaron_version": "0.18.0", + "group_id": null, + "artifact_id": "toga", + "version": "0.5.1", + "git_repo": "https://github.com/beeware/toga", + "git_tag": "ef1912b0a1b5c07793f9aa372409f5b9d36f2604", + "newline": "lf", + "language_version": [ + ">=3.9" + ], + "ecosystem": "pypi", + "purl": "pkg:pypi/toga@0.5.1", + "language": "python", + "build_tool": "pip", + "build_commands": [ + [ + "python", + "-m", + "build" + ] + ], + "build_backends": { + "setuptools": "==(80.3.1)", + "setuptools_scm": "==8.3.1", + "setuptools_dynamic_dependencies": "==1.0.0" + } +}