feat: add support to use inferred build tools and to extract tool-specific build dependency information (#1256)

Adds support to use inferred build tools and to extract tool-specific build dependency information for buildspec generation.

Signed-off-by: Abhinav Pradeep <[email protected]>

Author: AbhinavPradeep

src/macaron/build_spec_generator/common_spec/pypi_spec.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import re
+from typing import Any
 
 import tomli
 from packageurl import PackageURL
@@ -67,15 +68,17 @@ def get_default_build_commands(
 
             match build_tool_name:
                 case "pip":
-                    default_build_commands.append("python -m build".split())
+                    default_build_commands.append("python -m build --wheel -n".split())
                 case "poetry":
                     default_build_commands.append("poetry build".split())
                 case "flit":
+                    # We might also want to deal with existence flit.ini, we can do so via
+                    # "python -m flit.tomlify"
                     default_build_commands.append("flit build".split())
                 case "hatch":
                     default_build_commands.append("hatch build".split())
                 case "conda":
-                    default_build_commands.append("conda build".split())
+                    default_build_commands.append('echo("Not supported")'.split())
                 case _:
                     pass
 
@@ -156,6 +159,7 @@ def resolve_fields(self, purl: PackageURL) -> None:
                 try:
                     with pypi_package_json.sourcecode():
                         try:
+                            # Get the build time requirements from ["build-system", "requires"]
                             pyproject_content = pypi_package_json.get_sourcecode_file_contents("pyproject.toml")
                             content = tomli.loads(pyproject_content.decode("utf-8"))
                             requires = json_extract(content, ["build-system", "requires"], list)
@@ -164,10 +168,10 @@ def resolve_fields(self, purl: PackageURL) -> None:
                             backend = json_extract(content, ["build-system", "build-backend"], str)
                             if backend:
                                 build_backends_set.add(backend.replace(" ", ""))
-
                             python_version_constraint = json_extract(content, ["project", "requires-python"], str)
                             if python_version_constraint:
                                 python_version_set.add(python_version_constraint.replace(" ", ""))
+                            self.apply_tool_specific_inferences(build_requires_set, python_version_set, content)
                             logger.debug(
                                 "After analyzing pyproject.toml from the sdist: build-requires: %s, build_backend: %s",
                                 build_requires_set,
@@ -239,6 +243,40 @@ def resolve_fields(self, purl: PackageURL) -> None:
 
         self.data["build_commands"] = patched_build_commands
 
+    def apply_tool_specific_inferences(
+        self, build_requires_set: set[str], python_version_set: set[str], pyproject_contents: dict[str, Any]
+    ) -> None:
+        """
+        Based on build tools inferred, look into the pyproject.toml for related additional dependencies.
+
+        Parameters
+        ----------
+        build_requires_set: set[str]
+            Set of build requirements to populate.
+        python_version_set: set[str]
+            Set of compatible interpreter versions to populate.
+        pyproject_contents: dict[str, Any]
+            Parsed contents of the pyproject.toml file.
+        """
+        # If we have hatch as a build_tool, we will examine [tool.hatch.build.hooks.*] to
+        # look for any additional build dependencies declared there.
+        if "hatch" in self.data["build_tools"]:
+            # Look for [tool.hatch.build.hooks.*]
+            hatch_build_hooks = json_extract(pyproject_contents, ["tool", "hatch", "build", "hooks"], dict)
+            if hatch_build_hooks:
+                for _, section in hatch_build_hooks.items():
+                    dependencies = section.get("dependencies")
+                    if dependencies:
+                        build_requires_set.update(elem.replace(" ", "") for elem in dependencies)
+        # If we have flit as a build_tool, we will check if the legacy header [tool.flit.metadata] exists,
+        # and if so, check to see if we can use its "requires-python".
+        if "flit" in self.data["build_tools"]:
+            flit_python_version_constraint = json_extract(
+                pyproject_contents, ["tool", "flit", "metadata", "requires-python"], str
+            )
+            if flit_python_version_constraint:
+                python_version_set.add(flit_python_version_constraint.replace(" ", ""))
+
     def read_directory(self, wheel_path: str, purl: PackageURL) -> tuple[str, str]:
         """
         Read in the WHEEL and METADATA file from the .dist_info directory.

src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py

@@ -38,6 +38,17 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
         logger.debug("Could not derive a specific interpreter version.")
         raise GenerateBuildSpecError("Could not derive specific interpreter version.")
     backend_install_commands: str = " && ".join(build_backend_commands(buildspec))
+    build_tool_install: str = ""
+    if (
+        buildspec["build_tools"][0] != "pip"
+        and buildspec["build_tools"][0] != "conda"
+        and buildspec["build_tools"][0] != "flit"
+    ):
+        build_tool_install = f"pip install {buildspec['build_tools'][0]} && "
+    elif buildspec["build_tools"][0] == "flit":
+        build_tool_install = (
+            f"pip install {buildspec['build_tools'][0]} && if test -f \"flit.ini\"; then python -m flit.tomlify; fi && "
+        )
     dockerfile_content = f"""
     #syntax=docker/dockerfile:1.10
     FROM oraclelinux:9
@@ -87,7 +98,7 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
     EOF
 
     # Run the build
-    RUN /deps/bin/python -m build --wheel -n
+    RUN {"source /deps/bin/activate && " + build_tool_install + " ".join(x for x in buildspec["build_commands"][0])}
     """
 
     return dedent(dockerfile_content)
@@ -148,4 +159,6 @@ def build_backend_commands(buildspec: BaseBuildSpecDict) -> list[str]:
     commands: list[str] = []
     for backend, version_constraint in buildspec["build_requires"].items():
         commands.append(f'/deps/bin/pip install "{backend}{version_constraint}"')
+    # For a stable order on the install commands
+    commands.sort()
     return commands

tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr

@@ -50,7 +50,7 @@
   EOF
 
   # Run the build
-  RUN /deps/bin/python -m build --wheel -n
+  RUN source /deps/bin/activate && python -m build
 
   '''
 # ---

tests/build_spec_generator/dockerfile/compare_dockerfile_buildspec.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""Script to compare a generated dockerfile buildspec."""
+
+import argparse
+import logging
+from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logging.basicConfig(format="[%(filename)s:%(lineno)s %(tag)s] %(message)s")
+
+
+def log_with_tag(tag: str) -> Callable[[str], None]:
+    """Generate a log function that prints the name of the file and a tag at the beginning of each line."""
+
+    def log_fn(msg: str) -> None:
+        logger.info(msg, extra={"tag": tag})
+
+    return log_fn
+
+
+log_info = log_with_tag("INFO")
+log_err = log_with_tag("ERROR")
+log_passed = log_with_tag("PASSED")
+log_failed = log_with_tag("FAILED")
+
+
+def log_diff(result: str, expected: str) -> None:
+    """Pretty-print the diff of two strings."""
+    output = [
+        *("----  Result  ---", result),
+        *("---- Expected ---", expected),
+        "-----------------",
+    ]
+    log_info("\n".join(output))
+
+
+def main() -> int:
+    """Compare a Macaron generated dockerfile buildspec.
+
+    Returns
+    -------
+    int
+        0 if the generated dockerfile matches the expected output, or non-zero otherwise.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("result_dockerfile", help="the result dockerfile buildspec")
+    parser.add_argument("expected_dockerfile_buildspec", help="the expected buildspec dockerfile")
+    args = parser.parse_args()
+
+    # Load both files
+    with open(args.result_dockerfile, encoding="utf-8") as file:
+        buildspec = normalize(file.read())
+
+    with open(args.expected_dockerfile_buildspec, encoding="utf-8") as file:
+        expected_buildspec = normalize(file.read())
+
+    log_info(
+        f"Comparing the dockerfile buildspec {args.result_dockerfile} with the expected "
+        + "output dockerfile {args.expected_dockerfile_buildspec}"
+    )
+
+    # Compare the files
+    return compare(buildspec, expected_buildspec)
+
+
+def normalize(contents: str) -> list[str]:
+    """Convert string of file contents to list of its non-empty lines"""
+    return [line.strip() for line in contents.splitlines() if line.strip()]
+
+
+def compare(buildspec: list[str], expected_buildspec: list[str]) -> int:
+    """Compare the lines in the two files directly.
+
+    Early return when an unexpected difference is found. If the lengths
+    mismatch, but the first safe_index_max lines are the same, print
+    the missing/extra lines.
+
+    Returns
+    -------
+    int
+        0 if the generated dockerfile matches the expected output, or non-zero otherwise.
+    """
+    safe_index_max = min(len(buildspec), len(expected_buildspec))
+    for index in range(safe_index_max):
+        if buildspec[index] != expected_buildspec[index]:
+            # Log error
+            log_err("Mismatch found:")
+            # Log diff
+            log_diff(buildspec[index], expected_buildspec[index])
+            return 1
+    if safe_index_max < len(expected_buildspec):
+        log_err("Mismatch found: result is missing trailing lines")
+        log_diff("", "\n".join(expected_buildspec[safe_index_max:]))
+        return 1
+    if safe_index_max < len(buildspec):
+        log_err("Mismatch found: result has extra trailing lines")
+        log_diff("\n".join(buildspec[safe_index_max:]), "")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/integration/cases/pypi_cachetools/expected_default.buildspec

@@ -19,7 +19,9 @@
         [
             "python",
             "-m",
-            "build"
+            "build",
+            "--wheel",
+            "-n"
         ]
     ],
     "build_requires": {

tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec

@@ -0,0 +1,50 @@
+
+#syntax=docker/dockerfile:1.10
+FROM oraclelinux:9
+
+# Install core tools
+RUN dnf -y install which wget tar git
+
+# Install compiler and make
+RUN dnf -y install gcc make
+
+# Download and unzip interpreter
+RUN <<EOF
+    wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0.tgz
+    tar -xf Python-3.14.0.tgz
+EOF
+
+# Install necessary libraries to build the interpreter
+# From: https://devguide.python.org/getting-started/setup-building/
+RUN dnf install \
+  gcc-c++ gdb lzma glibc-devel libstdc++-devel openssl-devel \
+  readline-devel zlib-devel libzstd-devel libffi-devel bzip2-devel \
+  xz-devel sqlite sqlite-devel sqlite-libs libuuid-devel gdbm-libs \
+  perf expat expat-devel mpdecimal python3-pip
+
+# Build interpreter and create venv
+RUN <<EOF
+    cd Python-3.14.0
+    ./configure --with-pydebug
+    make -s -j $(nproc)
+    ./python -m venv /deps
+EOF
+
+# Clone code to rebuild
+RUN <<EOF
+    mkdir src
+    cd src
+    git clone https://github.com/tkem/cachetools .
+    git checkout --force ca7508fd56103a1b6d6f17c8e93e36c60b44ca25
+EOF
+
+WORKDIR /src
+
+# Install build and the build backends
+RUN <<EOF
+    /deps/bin/pip install "setuptools==80.9.0" && /deps/bin/pip install "wheel"
+    /deps/bin/pip install build
+EOF
+
+# Run the build
+RUN source /deps/bin/activate && python -m build --wheel -n

tests/integration/cases/pypi_cachetools/test.yaml

@@ -30,3 +30,17 @@ steps:
     kind: default_build_spec
     result: output/buildspec/pypi/cachetools/macaron.buildspec
     expected: expected_default.buildspec
+- name: Generate the buildspec
+  kind: gen-build-spec
+  options:
+    command_args:
+    - -purl
+    - pkg:pypi/[email protected]
+    - --output-format
+    - dockerfile
+- name: Compare Dockerfile.
+  kind: compare
+  options:
+    kind: dockerfile_build_spec
+    result: output/buildspec/pypi/cachetools/dockerfile.buildspec
+    expected: expected_dockerfile.buildspec

tests/integration/cases/pypi_markdown-it-py/expected_dockerfile.buildspec

@@ -0,0 +1,50 @@
+
+#syntax=docker/dockerfile:1.10
+FROM oraclelinux:9
+
+# Install core tools
+RUN dnf -y install which wget tar git
+
+# Install compiler and make
+RUN dnf -y install gcc make
+
+# Download and unzip interpreter
+RUN <<EOF
+    wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0.tgz
+    tar -xf Python-3.14.0.tgz
+EOF
+
+# Install necessary libraries to build the interpreter
+# From: https://devguide.python.org/getting-started/setup-building/
+RUN dnf install \
+  gcc-c++ gdb lzma glibc-devel libstdc++-devel openssl-devel \
+  readline-devel zlib-devel libzstd-devel libffi-devel bzip2-devel \
+  xz-devel sqlite sqlite-devel sqlite-libs libuuid-devel gdbm-libs \
+  perf expat expat-devel mpdecimal python3-pip
+
+# Build interpreter and create venv
+RUN <<EOF
+    cd Python-3.14.0
+    ./configure --with-pydebug
+    make -s -j $(nproc)
+    ./python -m venv /deps
+EOF
+
+# Clone code to rebuild
+RUN <<EOF
+    mkdir src
+    cd src
+    git clone https://github.com/executablebooks/markdown-it-py .
+    git checkout --force c62983f1554124391b47170180e6c62df4d476ca
+EOF
+
+WORKDIR /src
+
+# Install build and the build backends
+RUN <<EOF
+    /deps/bin/pip install "flit==3.12.0" && /deps/bin/pip install "flit_core<4,>=3.4"
+    /deps/bin/pip install build
+EOF
+
+# Run the build
+RUN source /deps/bin/activate && pip install flit && if test -f "flit.ini"; then python -m flit.tomlify; fi && flit build

tests/integration/cases/pypi_markdown-it-py/test.yaml

@@ -27,3 +27,17 @@ steps:
     kind: default_build_spec
     result: output/buildspec/pypi/markdown-it-py/macaron.buildspec
     expected: expected_default.buildspec
+- name: Generate the buildspec
+  kind: gen-build-spec
+  options:
+    command_args:
+    - -purl
+    - pkg:pypi/[email protected]
+    - --output-format
+    - dockerfile
+- name: Compare Dockerfile
+  kind: compare
+  options:
+    kind: dockerfile_build_spec
+    result: output/buildspec/pypi/markdown-it-py/dockerfile.buildspec
+    expected: expected_dockerfile.buildspec

tests/integration/cases/pypi_toga/expected_default.buildspec

@@ -19,7 +19,9 @@
         [
             "python",
             "-m",
-            "build"
+            "build",
+            "--wheel",
+            "-n"
         ]
     ],
     "build_requires": {