Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a CI test for the dbt-metricflow package #1653

Merged
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
/* PR_START p--release-dbt-metricflow-0.8.1-updates 04 */ For package…
… tests, activate python venv via shell to handle PATH more easily.
plypaul committed Jan 30, 2025
commit 6b7ed774b86c9e73bd69c4d30895d9ee0e36d395
57 changes: 41 additions & 16 deletions scripts/ci_tests/run_package_build_tests.py
Original file line number Diff line number Diff line change
@@ -8,14 +8,17 @@
import logging
import tempfile
import venv
from collections.abc import Sequence
from pathlib import Path

from scripts.mf_script_helper import MetricFlowScriptHelper

logger = logging.getLogger(__name__)


def _run_package_build_test(package_directory: Path, package_test_script: Path) -> None:
def _run_package_build_test(
package_directory: Path, package_test_script: Path, optional_package_dependencies_to_install: Sequence[str] = ()
) -> None:
"""Run a test to verify that a package is built properly.

Given the directory where the package is located, this will build the package using `hatch build` and install the
@@ -25,40 +28,62 @@ def _run_package_build_test(package_directory: Path, package_test_script: Path)
Args:
package_directory: Root directory where the package is located.
package_test_script: The path to the script that should be run.

optional_package_dependencies_to_install: If the given package defines optional dependencies that can be
installed, install these. e.g. for `dbt-metricflow[dbt-duckdb]`, specify `dbt-duckdb`.
Returns: None
Raises: Exception on test failure.
"""
logger.info(f"Running package build test for {str(package_directory)!r} using {str(package_test_script)!r}")
package_directory_str = package_directory.as_posix()
package_test_script_str = package_test_script.as_posix()
logger.info(f"Running package build test for {package_directory_str!r} using {package_test_script_str!r}")

try:
with tempfile.TemporaryDirectory() as temporary_directory_str:
temporary_directory = Path(temporary_directory_str)
venv_directory = temporary_directory.joinpath("venv")
logger.info(f"Creating venv at {str(venv_directory)!r}")
logger.info(f"Creating a new venv at {venv_directory.as_posix()!r}")

venv.create(venv_directory, with_pip=True)
pip_executable = Path(venv_directory, "bin/pip")
python_executable = Path(venv_directory, "bin/python")
pip_executable = Path(venv_directory, "bin/pip").as_posix()

logger.info(f"Building package at {str(package_directory)!r}")
logger.info(f"Running package build test for {str(package_directory)!r} using {str(package_test_script)!r}")
logger.info(f"Building package at {package_directory_str!r}")
MetricFlowScriptHelper.run_command(["hatch", "clean"], working_directory=package_directory)
MetricFlowScriptHelper.run_command(["hatch", "build"], working_directory=package_directory)

logger.info("Installing package using generated wheels")
MetricFlowScriptHelper.run_shell_command(f'{pip_executable} install "{str(package_directory)}"/dist/*.whl')

logger.info("Running test using installed package in venv")
MetricFlowScriptHelper.run_command(
[str(python_executable), str(package_test_script)], working_directory=temporary_directory
logger.info("Installing package in venv using generated wheels")
paths_to_wheels = _get_wheels_in_directory(package_directory.joinpath("dist"))
if len(paths_to_wheels) != 1:
raise RuntimeError(f"Expected exactly one wheel but got {paths_to_wheels}")

path_to_wheel = paths_to_wheels[0]
MetricFlowScriptHelper.run_command([pip_executable, "install", path_to_wheel.as_posix()])
for optional_package_dependency in optional_package_dependencies_to_install:
MetricFlowScriptHelper.run_command(
[pip_executable, "install", f"{path_to_wheel.as_posix()}[{optional_package_dependency}]"]
)

logger.info("Running test using venv")
venv_activate = venv_directory.joinpath("bin", "activate").as_posix()
MetricFlowScriptHelper.run_shell_command(
# Using period instead of `source` for compatibility with `sh`.
f"cd {temporary_directory_str} && . {venv_activate} && python {package_test_script_str}",
)

logger.info(f"Test passed {str(package_test_script)!r}")
logger.info(f"Test passed {package_test_script_str!r}")
except Exception as e:
raise PackageBuildTestFailureException(
f"Package build test failed for {str(package_directory)!r} using {str(package_test_script)!r}"
f"Package build test failed for {package_directory_str!r} using {package_test_script_str!r}"
) from e


def _get_wheels_in_directory(directory: Path) -> Sequence[Path]:
paths_to_wheels = []
for path_item in directory.iterdir():
if path_item.is_file() and path_item.suffix == ".whl":
paths_to_wheels.append(path_item)
return paths_to_wheels


class PackageBuildTestFailureException(Exception): # noqa: D101
pass