diff --git a/.github/workflows/publish-py-sdk-canary-scheduler.yaml b/.github/workflows/publish-py-sdk-canary-scheduler.yaml new file mode 100644 index 00000000..6664711c --- /dev/null +++ b/.github/workflows/publish-py-sdk-canary-scheduler.yaml @@ -0,0 +1,47 @@ +# This workflow schedules the Python SDK canary publish workflow on a daily basis. +# The actual publish still happens in `test-publish-py-sdk.yaml` so TestPyPI trusted +# publishing only needs that one workflow configured as the publisher. + +name: Schedule Python SDK Canary Publish + +concurrency: + group: publish-py-sdk-canary-scheduler + cancel-in-progress: false + +on: + schedule: + - cron: "23 6 * * *" + workflow_dispatch: + +jobs: + dispatch-canary-publish: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Dispatch TestPyPI publish workflow + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "test-publish-py-sdk.yaml", + ref: "main", + inputs: { + ref: "main", + release_type: "canary", + dry_run: "false", + }, + }); + + - name: Summarize dispatch + run: | + { + echo "## Python SDK Canary Dispatch Queued" + echo + echo "- Workflow: \`test-publish-py-sdk.yaml\`" + echo "- Release type: \`canary\`" + echo "- Ref: \`main\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test-publish-py-sdk.yaml b/.github/workflows/test-publish-py-sdk.yaml index 8f0e5967..4a7e9fe4 100644 --- a/.github/workflows/test-publish-py-sdk.yaml +++ b/.github/workflows/test-publish-py-sdk.yaml @@ -1,8 +1,6 @@ -# # This workflow is used to publish the Python SDK to TestPyPI. # It mirrors the main release workflow where practical, but it does not create git tags # or GitHub Releases. -# name: Publish Python SDK to TestPyPI @@ -14,6 +12,14 @@ on: required: true type: string default: "main" + release_type: + description: "Release type to publish to TestPyPI" + required: true + type: choice + options: + - prerelease + - canary + default: prerelease dry_run: description: "Validate and build without publishing to TestPyPI" required: true @@ -27,6 +33,9 @@ jobs: outputs: commit_sha: ${{ steps.validate.outputs.commit_sha }} dry_run: ${{ steps.validate.outputs.dry_run }} + release_type: ${{ steps.validate.outputs.release_type }} + target_branch: ${{ steps.validate.outputs.target_branch }} + version: ${{ steps.validate.outputs.version }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -37,10 +46,20 @@ jobs: with: cache: true experimental: true - - name: Resolve commit + - name: Resolve test release metadata id: validate run: | - echo "commit_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + RELEASE_TYPE="${{ github.event.inputs.release_type }}" + if [ "$RELEASE_TYPE" = "canary" ]; then + mise exec -- python py/scripts/test-publish.py prepare canary \ + "${{ github.event.inputs.ref }}" \ + --run-number "${{ github.run_number }}" \ + --github-output "$GITHUB_OUTPUT" + else + mise exec -- python py/scripts/test-publish.py prepare prerelease \ + --run-number "${{ github.run_number }}" \ + --github-output "$GITHUB_OUTPUT" + fi echo "dry_run=${{ github.event.inputs.dry_run }}" >> "$GITHUB_OUTPUT" build-and-publish: @@ -48,15 +67,23 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 permissions: + actions: read id-token: write # Required for PyPI trusted publishing outputs: - version: ${{ steps.get_version.outputs.version }} + commit_sha: ${{ steps.publish_status.outputs.commit_sha }} + package_name: ${{ steps.publish_status.outputs.package_name }} + published: ${{ steps.publish_status.outputs.published }} + reason: ${{ steps.publish_status.outputs.reason }} + version: ${{ steps.publish_status.outputs.version }} env: COMMIT_SHA: ${{ needs.validate.outputs.commit_sha }} DRY_RUN: ${{ needs.validate.outputs.dry_run }} PYPI_REPO: testpypi + RELEASE_TYPE: ${{ needs.validate.outputs.release_type }} + TARGET_BRANCH: ${{ needs.validate.outputs.target_branch }} + VERSION: ${{ needs.validate.outputs.version }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -68,23 +95,66 @@ jobs: with: cache: true experimental: true + - name: Check whether a new canary is needed + if: env.RELEASE_TYPE == 'canary' + id: should_publish + run: | + mise exec -- python py/scripts/test-publish.py check-canary --github-output "$GITHUB_OUTPUT" + - name: Check Python CI status + if: env.RELEASE_TYPE == 'canary' && steps.should_publish.outputs.should_publish == 'true' + id: ci_status + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + TARGET_BRANCH: ${{ env.TARGET_BRANCH }} + with: + script: | + const { owner, repo } = context.repo; + const response = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: "py.yaml", + branch: process.env.TARGET_BRANCH, + status: "completed", + per_page: 1, + }); + + const run = response.data.workflow_runs[0]; + if (!run) { + core.setOutput("should_publish", "false"); + core.setOutput("reason", `No completed py.yaml run found on ${process.env.TARGET_BRANCH}.`); + return; + } + + if (run.conclusion !== "success") { + core.setOutput("should_publish", "false"); + core.setOutput( + "reason", + `Latest completed py.yaml run on ${process.env.TARGET_BRANCH} concluded with ${run.conclusion} (${run.html_url}).`, + ); + return; + } + + core.setOutput("should_publish", "true"); + core.setOutput( + "reason", + `Latest completed py.yaml run on ${process.env.TARGET_BRANCH} succeeded (${run.html_url}).`, + ); - name: Build and verify + if: env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true') + env: + BRAINTRUST_RELEASE_CHANNEL: ${{ env.RELEASE_TYPE }} + BRAINTRUST_VERSION_OVERRIDE: ${{ env.VERSION }} run: | mise exec -- make -C py install-dev verify-build - name: Upload build artifacts + if: env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true') uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: python-sdk-testpypi-dist path: py/dist/ retention-days: 5 - - name: Get version from built wheel - id: get_version - run: | - WHEEL=$(ls py/dist/*.whl | head -n 1) - VERSION=$(echo "$WHEEL" | sed -n 's/.*braintrust-\([^-]*\)-.*/\1/p') - echo "version=$VERSION" >> $GITHUB_OUTPUT - name: Publish to TestPyPI - if: env.DRY_RUN != 'true' + if: env.DRY_RUN != 'true' && (env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true')) uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 with: repository-url: https://test.pypi.org/legacy/ @@ -92,31 +162,146 @@ jobs: - name: Summarize dry run if: env.DRY_RUN == 'true' run: | - echo "Dry run completed for TestPyPI build from $COMMIT_SHA" + { + echo "## Python SDK TestPyPI Dry Run Completed" + echo + echo "- Release type: \`${RELEASE_TYPE}\`" + echo "- Version: \`${VERSION}\`" + echo "- Commit: \`${COMMIT_SHA}\`" + } >> "$GITHUB_STEP_SUMMARY" + - name: Summarize skipped canary publish + if: env.RELEASE_TYPE == 'canary' && env.DRY_RUN != 'true' && (steps.should_publish.outputs.should_publish != 'true' || steps.ci_status.outputs.should_publish != 'true') + env: + SHOULD_PUBLISH_REASON: ${{ steps.should_publish.outputs.reason }} + CI_REASON: ${{ steps.ci_status.outputs.reason }} + run: | + set -euo pipefail + REASON="${CI_REASON:-$SHOULD_PUBLISH_REASON}" + { + echo "## Python SDK Canary Publish Skipped" + echo + echo "$REASON" + } >> "$GITHUB_STEP_SUMMARY" + - name: Set publish status outputs + id: publish_status + if: always() + env: + SHOULD_PUBLISH: ${{ steps.should_publish.outputs.should_publish }} + CI_SHOULD_PUBLISH: ${{ steps.ci_status.outputs.should_publish }} + SHOULD_PUBLISH_REASON: ${{ steps.should_publish.outputs.reason }} + CI_REASON: ${{ steps.ci_status.outputs.reason }} + run: | + set -euo pipefail + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "package_name=braintrust" >> "$GITHUB_OUTPUT" + echo "commit_sha=$(git rev-parse --short=7 HEAD)" >> "$GITHUB_OUTPUT" + + if [ "${DRY_RUN}" = "true" ]; then + echo "published=false" >> "$GITHUB_OUTPUT" + echo "reason=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "${RELEASE_TYPE}" != "canary" ]; then + echo "published=true" >> "$GITHUB_OUTPUT" + echo "reason=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "${SHOULD_PUBLISH}" = "true" ] && [ "${CI_SHOULD_PUBLISH}" = "true" ]; then + echo "published=true" >> "$GITHUB_OUTPUT" + echo "reason=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + REASON="${CI_REASON:-$SHOULD_PUBLISH_REASON}" + echo "published=false" >> "$GITHUB_OUTPUT" + echo "reason=${REASON}" >> "$GITHUB_OUTPUT" notify-success: needs: [validate, build-and-publish] - if: always() && needs.build-and-publish.result == 'success' + if: always() && needs.build-and-publish.result == 'success' && (needs.validate.outputs.dry_run == 'true' || needs.build-and-publish.outputs.published == 'true') + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Post to Slack on dry run success + if: needs.validate.outputs.dry_run == 'true' + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: C0ABHT0SWA2 + text: "🧪 Python SDK TestPyPI dry run succeeded" + blocks: + - type: "header" + text: + type: "plain_text" + text: "🧪 Python SDK TestPyPI Dry Run Succeeded" + - type: "section" + text: + type: "mrkdwn" + text: "${{ format('*Mode:* dry run\n*Release type:* {0}\n*Version:* {1}\n*Ref:* {2}\n*Commit:* {3}\n\n<{4}/{5}/actions/runs/{6}|View Run>', needs.validate.outputs.release_type, needs.validate.outputs.version, github.event.inputs.ref, needs.validate.outputs.commit_sha, github.server_url, github.repository, github.run_id) }}" + - name: Post to Slack on prerelease success + if: needs.validate.outputs.dry_run != 'true' && needs.validate.outputs.release_type == 'prerelease' + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: C0ABHT0SWA2 + text: "🧪 Python SDK prerelease v${{ needs.build-and-publish.outputs.version }} published to TestPyPI" + blocks: + - type: "header" + text: + type: "plain_text" + text: "🧪 Python SDK Pre-release Published" + - type: "section" + text: + type: "mrkdwn" + text: "${{ format('*Version:* {0}\n*Ref:* {1}\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* \n\n<{2}/{3}/actions/runs/{4}|View Run>', needs.build-and-publish.outputs.version, github.event.inputs.ref, github.server_url, github.repository, github.run_id) }}" + - name: Post to Slack on canary success + if: needs.validate.outputs.dry_run != 'true' && needs.validate.outputs.release_type == 'canary' + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: C0ABHT0SWA2 + text: "🧪 Python SDK canary ${{ needs.build-and-publish.outputs.version }} published to TestPyPI" + blocks: + - type: "header" + text: + type: "plain_text" + text: "🧪 Python SDK Canary Published" + - type: "section" + text: + type: "mrkdwn" + text: "${{ format('*Version:* {0}\n*Branch:* `{1}`\n*Commit:* `{2}`\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* \n\n<{3}/{4}/actions/runs/{5}|View Run>', needs.build-and-publish.outputs.version, needs.validate.outputs.target_branch, needs.build-and-publish.outputs.commit_sha, github.server_url, github.repository, github.run_id) }}" + + notify-skipped: + needs: [validate, build-and-publish] + if: always() && needs.validate.outputs.release_type == 'canary' && needs.validate.outputs.dry_run != 'true' && needs.build-and-publish.result == 'success' && needs.build-and-publish.outputs.published != 'true' runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Post to Slack on success + - name: Post to Slack on intentional skip uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} payload: | channel: C0ABHT0SWA2 - text: "${{ needs.validate.outputs.dry_run == 'true' && '🧪 Python SDK TestPyPI dry run succeeded' || format('🧪 Python SDK prerelease v{0} published to TestPyPI', needs.build-and-publish.outputs.version) }}" + text: "⏭️ Python SDK canary publish skipped: ${{ needs.build-and-publish.outputs.reason }}" blocks: - type: "header" text: type: "plain_text" - text: "${{ needs.validate.outputs.dry_run == 'true' && '🧪 Python SDK TestPyPI Dry Run Succeeded' || '🧪 Python SDK Pre-release Published' }}" + text: "⏭️ Python SDK Canary Publish Skipped" - type: "section" text: type: "mrkdwn" - text: "${{ needs.validate.outputs.dry_run == 'true' && format('*Mode:* dry run\n*Ref:* {0}\n*Commit:* {1}\n\n<{2}/{3}/actions/runs/{4}|View Run>', github.event.inputs.ref, needs.validate.outputs.commit_sha, github.server_url, github.repository, github.run_id) || format('*Version:* {0}\n*Ref:* {1}\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* \n\n<{2}/{3}/actions/runs/{4}|View Run>', needs.build-and-publish.outputs.version, github.event.inputs.ref, github.server_url, github.repository, github.run_id) }}" + text: "${{ format('*Branch:* `{0}`\n*Commit:* `{1}`\n*Reason:* {2}\n\n<{3}/{4}/actions/runs/{5}|View Run>', needs.validate.outputs.target_branch, needs.build-and-publish.outputs.commit_sha, needs.build-and-publish.outputs.reason, github.server_url, github.repository, github.run_id) }}" notify-failure: needs: [validate, build-and-publish] diff --git a/AGENTS.md b/AGENTS.md index 5f5f199e..2744c5ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -133,7 +133,7 @@ make build Important caveat: -- `py/scripts/template-version.sh` rewrites `py/src/braintrust/version.py` during build. +- `py/scripts/template-version.py` rewrites `py/src/braintrust/version.py` during build. - `py/Makefile` restores that file afterward with `git checkout`. Avoid editing `py/src/braintrust/version.py` while also running build commands. diff --git a/docs/publishing.md b/docs/publishing.md index 13c4e809..c1cf0adc 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -61,20 +61,43 @@ This is useful for: - validating a release candidate before the real PyPI publish - sharing prerelease artifacts for testing without consuming the final PyPI version number -The workflow reads the version from `py/src/braintrust/version.py`, then appends `rc` during the build when publishing to TestPyPI. For example, `0.8.0` becomes something like `0.8.0rc1234`. +The workflow reads the version from `py/src/braintrust/version.py` and applies a workflow-controlled version override during the build so TestPyPI uploads stay unique without modifying the checked-in file. The packaged `version.py` is also templated with the exact git commit and a release channel marker. + +It supports two release types: + +- `prerelease`: keeps the existing TestPyPI prerelease behavior and publishes a version such as `0.8.0rc1234` +- `canary`: publishes a nightly-style development release to TestPyPI only Run `Publish Python SDK to TestPyPI` with: - `ref=main` or the exact branch / commit you want to test +- `release_type=prerelease` or `release_type=canary` - `dry_run=true` if you only want to validate/build without publishing +### Canary releases + +- Can be triggered manually by running `Publish Python SDK to TestPyPI` with `release_type=canary` +- Publish a PEP 440 development release in the form `.dev` +- Only publish to TestPyPI; there is no matching canary mode in the real PyPI workflow +- Do not create a git tag or GitHub Release +- Skip publishing if the current `HEAD` commit matches the latest published TestPyPI artifact marked with release channel `canary` +- Skip publishing unless the latest completed `py.yaml` run on the target branch succeeded + +install canaries like so: + +```bash +pip install -i https://test.pypi.org/simple/ braintrust== +``` + +Nightly scheduling lives in `Schedule Python SDK Canary Publish`, which only dispatches `Publish Python SDK to TestPyPI` with `release_type=canary`. The actual publish remains in `test-publish-py-sdk.yaml` so trusted publishing stays configured against a single workflow. + Install from TestPyPI with: ```bash pip install -i https://test.pypi.org/simple/ braintrust== ``` -> The build will fail if you upload a package with a duplicate version number. If this happens, DO NOT update version.py. Instead, rebase your branch onto origin/main and try again. The workflow will add an incrementing suffix rc. So as long as you are up to date, this should just work. +> The build will fail if you upload a package with a duplicate version number. If this happens, DO NOT update version.py. Instead, rebase your branch onto origin/main and try again. The workflow-generated prerelease or canary suffix should normally keep TestPyPI versions unique. Just like the main PyPI workflow, the TestPyPI workflow also supports `dry_run=true`. In that mode it builds, verifies, and uploads artifacts, but it does not publish to TestPyPI. diff --git a/py/Makefile b/py/Makefile index f692d4e0..4696d84d 100644 --- a/py/Makefile +++ b/py/Makefile @@ -33,7 +33,7 @@ test-core: nox -s test_core _template-version: - @bash scripts/template-version.sh + @$(PYTHON) scripts/template-version.py build: clean _template-version $(PYTHON) -m build diff --git a/py/noxfile.py b/py/noxfile.py index 78f9cffb..4c25af1f 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -12,12 +12,29 @@ import glob import os +import pathlib +import re import sys import tempfile import nox +def _pinned_python_version(): + """Return the (major, minor) Python version pinned in ../.tool-versions, or None.""" + tool_versions = pathlib.Path(__file__).parent.parent / ".tool-versions" + try: + for line in tool_versions.read_text().splitlines(): + m = re.match(r"^python\s+(\d+)\.(\d+)", line) + if m: + return (int(m.group(1)), int(m.group(2))) + except OSError: + pass + return None + + +_PINNED_PYTHON = _pinned_python_version() + # much faster than pip nox.options.default_venv_backend = "uv" @@ -298,9 +315,6 @@ def test_otel_not_installed(session): @nox.session() def pylint(session): # pylint needs everything so we don't trigger missing import errors - # Skip on Python < 3.10 because some deps (like temporalio 1.19+) require 3.10+ - if sys.version_info < (3, 10): - session.skip("pylint requires Python >= 3.10 for full dependency support") session.install(".[all]") session.install("-r", "requirements-dev.txt") session.install(*VENDOR_PACKAGES) @@ -316,6 +330,10 @@ def pylint(session): files = [path for path in result.strip().splitlines() if path not in GENERATED_LINT_EXCLUDES] if not files: return + # scripts/ may use APIs only available in the latest pinned Python version + # (e.g. datetime.UTC requires 3.11+); skip them on older versions. + if _PINNED_PYTHON and sys.version_info[:2] < _PINNED_PYTHON: + files = [f for f in files if not f.startswith("scripts/")] session.run("pylint", "--errors-only", *files) diff --git a/py/scripts/template-version.py b/py/scripts/template-version.py new file mode 100644 index 00000000..72a3a300 --- /dev/null +++ b/py/scripts/template-version.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Template the package version module during build.""" + +import os +import pathlib +import re +import subprocess + + +VERSION_FILE = pathlib.Path("src/braintrust/version.py") +GIT_COMMIT_RE = re.compile(r"__GIT_COMMIT__") +RELEASE_CHANNEL_RE = re.compile(r'^RELEASE_CHANNEL = ".*"$', re.MULTILINE) +VERSION_RE = re.compile(r'^VERSION = ".*"$', re.MULTILINE) + + +def run(*args: str) -> str: + result = subprocess.run(args, check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def main() -> int: + contents = VERSION_FILE.read_text(encoding="utf-8") + git_commit = run("git", "rev-parse", "HEAD") + contents = GIT_COMMIT_RE.sub(git_commit, contents) + release_channel = os.environ.get("BRAINTRUST_RELEASE_CHANNEL") + if release_channel: + contents = RELEASE_CHANNEL_RE.sub(f'RELEASE_CHANNEL = "{release_channel}"', contents, count=1) + + version_override = os.environ.get("BRAINTRUST_VERSION_OVERRIDE") + if version_override: + contents = VERSION_RE.sub(f'VERSION = "{version_override}"', contents, count=1) + elif os.environ.get("PYPI_REPO") == "testpypi" and os.environ.get("GITHUB_RUN_NUMBER"): + current_version_match = re.search(r'^VERSION = "([^"]+)"$', contents, re.MULTILINE) + if current_version_match is None: + raise ValueError(f"Could not find VERSION in {VERSION_FILE}") + new_version = f"{current_version_match.group(1)}rc{os.environ['GITHUB_RUN_NUMBER']}" + contents = VERSION_RE.sub(f'VERSION = "{new_version}"', contents, count=1) + + VERSION_FILE.write_text(contents, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/py/scripts/template-version.sh b/py/scripts/template-version.sh deleted file mode 100755 index f419ee74..00000000 --- a/py/scripts/template-version.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -e - -VERSION_FILE="src/braintrust/version.py" - -GIT_COMMIT=$(git rev-parse HEAD) - -sed_inplace() { - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "$@" - else - sed -i "$@" - fi -} - -# Update git commit hash -sed_inplace "s/__GIT_COMMIT__/$GIT_COMMIT/g" "$VERSION_FILE" - -# Get current version -CURRENT_VERSION=$(grep 'VERSION = ' "$VERSION_FILE" | cut -d'"' -f2) - -# If we're uploading to testpypi, add a run number to the version so we can -# test multiple times. -if [[ "$PYPI_REPO" == "testpypi" ]] && [[ -n "$GITHUB_RUN_NUMBER" ]]; then - NEW_VERSION="${CURRENT_VERSION}rc${GITHUB_RUN_NUMBER}" - sed_inplace "s/VERSION = \".*\"/VERSION = \"$NEW_VERSION\"/" "$VERSION_FILE" -fi diff --git a/py/scripts/test-publish.py b/py/scripts/test-publish.py new file mode 100644 index 00000000..9f10dffa --- /dev/null +++ b/py/scripts/test-publish.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Support scripts for TestPyPI release workflows.""" + +from __future__ import annotations + +import argparse +import datetime +import io +import json +import pathlib +import re +import subprocess +import sys +import tarfile +import urllib.error +import urllib.request +import zipfile +from typing import Any + + +try: + from packaging.version import InvalidVersion, Version +except ImportError: # pragma: no cover + from pkg_resources import parse_version + + class InvalidVersion(Exception): + """Compatibility shim when packaging is unavailable.""" + + def Version(version: str) -> Any: # type: ignore[misc] + return parse_version(version) + + +STABLE_VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +PRERELEASE_VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(a|b|rc)[0-9]+$") +GIT_COMMIT_RE = re.compile(r'^GIT_COMMIT = "([0-9a-f]{40})"$', re.MULTILINE) +RELEASE_CHANNEL_RE = re.compile(r'^RELEASE_CHANNEL = "([^"]+)"$', re.MULTILINE) +HTTP_TIMEOUT_SECONDS = 30 +USER_AGENT = "braintrust-test-publish" + + +def run(*args: str) -> str: + result = subprocess.run(args, check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def get_repo_root() -> pathlib.Path: + return pathlib.Path(run("git", "rev-parse", "--show-toplevel")) + + +def get_version(repo_root: pathlib.Path) -> str: + version_file = repo_root / "py" / "src" / "braintrust" / "version.py" + version_line = next( + (line for line in version_file.read_text(encoding="utf-8").splitlines() if line.startswith("VERSION = ")), + None, + ) + if version_line is None: + raise ValueError(f"Could not find VERSION in {version_file}") + return version_line.split('"')[1] + + +def write_github_output(output_path: pathlib.Path, values: dict[str, str]) -> None: + with output_path.open("a", encoding="utf-8") as handle: + for key, value in values.items(): + handle.write(f"{key}={value}\n") + + +def make_request(url: str) -> urllib.request.Request: + return urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + + +def urlopen(url: str): + return urllib.request.urlopen(make_request(url), timeout=HTTP_TIMEOUT_SECONDS) + + +def sanitize_run_number(run_number: int) -> int: + if run_number <= 0: + raise ValueError(f"run_number must be positive; found '{run_number}'") + return run_number + + +def build_prerelease_version(base_version: str, run_number: int) -> str: + if STABLE_VERSION_RE.fullmatch(base_version): + return f"{base_version}rc{run_number}" + if PRERELEASE_VERSION_RE.fullmatch(base_version): + match = re.fullmatch(r"(?P[0-9]+\.[0-9]+\.[0-9]+(?:a|b|rc))(?P[0-9]+)", base_version) + if match is None: + raise ValueError(f"Could not parse prerelease version '{base_version}'") + return f"{match.group('prefix')}{run_number}" + raise ValueError(f"TestPyPI prereleases require a base version like X.Y.Z or X.Y.Zrc1; found '{base_version}'") + + +def check_testpypi_version_does_not_exist(version: str) -> None: + url = f"https://test.pypi.org/pypi/braintrust/{version}/json" + try: + with urlopen(url) as response: + if response.status == 200: + raise ValueError(f"Version '{version}' already exists on TestPyPI") + raise ValueError( + f"Unexpected response from TestPyPI while checking version '{version}' (HTTP {response.status})" + ) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return + raise ValueError( + f"Unexpected response from TestPyPI while checking version '{version}' (HTTP {exc.code})" + ) from exc + except urllib.error.URLError as exc: + raise ValueError(f"Failed to reach TestPyPI while checking version '{version}': {exc.reason}") from exc + + +def prepare_release(release_type: str, target_ref: str, run_number: int) -> dict[str, str]: + repo_root = get_repo_root() + base_version = get_version(repo_root) + commit_sha = run("git", "rev-parse", "HEAD") + target_branch = "" + run_number = sanitize_run_number(run_number) + + if release_type == "prerelease": + version = build_prerelease_version(base_version, run_number) + elif release_type == "canary": + if not (STABLE_VERSION_RE.fullmatch(base_version) or PRERELEASE_VERSION_RE.fullmatch(base_version)): + raise ValueError(f"Canaries require a base version like X.Y.Z or X.Y.Zrc1; found '{base_version}'") + branch_check = subprocess.run( + ("git", "ls-remote", "--exit-code", "--heads", "origin", target_ref), + check=False, + capture_output=True, + text=True, + ) + if branch_check.returncode != 0: + raise ValueError( + f"Canary releases must target a branch ref that exists on origin; '{target_ref}' is not a branch" + ) + target_branch = target_ref + date_stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%d") + version = f"{base_version}.dev{date_stamp}{run_number:06d}" + else: + raise ValueError(f"Unsupported release type '{release_type}'") + + check_testpypi_version_does_not_exist(version) + return { + "commit_sha": commit_sha, + "release_type": release_type, + "target_branch": target_branch, + "version": version, + } + + +def load_testpypi_project_json() -> dict[str, Any]: + try: + with urlopen("https://test.pypi.org/pypi/braintrust/json") as response: + return json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return {"releases": {}} + raise + + +def extract_version_module_from_distribution(file_url: str) -> str: + with urlopen(file_url) as response: + payload = response.read() + + if file_url.endswith(".whl"): + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + version_member = next(name for name in archive.namelist() if name.endswith("braintrust/version.py")) + return archive.read(version_member).decode("utf-8") + elif file_url.endswith((".tar.gz", ".tgz")): + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive: + version_member = next( + member for member in archive.getmembers() if member.name.endswith("src/braintrust/version.py") + ) + extracted = archive.extractfile(version_member) + if extracted is None: + raise ValueError(f"Could not read {version_member.name} from {file_url}") + return extracted.read().decode("utf-8") + else: + raise ValueError(f"Unsupported distribution file for canary inspection: {file_url}") + + +def extract_release_metadata_from_distribution(file_url: str) -> tuple[str, str]: + contents = extract_version_module_from_distribution(file_url) + commit_match = GIT_COMMIT_RE.search(contents) + release_channel_match = RELEASE_CHANNEL_RE.search(contents) + if commit_match is None: + raise ValueError(f"Could not find templated GIT_COMMIT in {file_url}") + if release_channel_match is None: + raise ValueError(f"Could not find templated RELEASE_CHANNEL in {file_url}") + return commit_match.group(1), release_channel_match.group(1) + + +def inspect_release_metadata(version: str, files: list[dict[str, Any]]) -> tuple[str, str]: + preferred_file = next((file for file in files if file["filename"].endswith(".whl")), files[0]) + try: + return extract_release_metadata_from_distribution(preferred_file["url"]) + except ( + StopIteration, + KeyError, + LookupError, + OSError, + ValueError, + zipfile.BadZipFile, + tarfile.TarError, + urllib.error.URLError, + ) as exc: + raise ValueError(f"Could not inspect TestPyPI release {version}: {exc}") from exc + + +def check_canary_status() -> dict[str, str]: + current_commit = run("git", "rev-parse", "HEAD") + current_short = run("git", "rev-parse", "--short=7", "HEAD") + releases = load_testpypi_project_json().get("releases", {}) + candidates: list[tuple[str, Any]] = [] + for version, files in releases.items(): + if not files: + continue + try: + candidates.append((version, Version(version))) + except InvalidVersion: + continue + candidates.sort(key=lambda item: item[1], reverse=True) + + latest_canary_version = "" + latest_canary_commit = "" + ignored_release_count = 0 + + for version, _parsed_version in candidates: + try: + previous_commit, release_channel = inspect_release_metadata(version, releases[version]) + except ValueError: + ignored_release_count += 1 + continue + if release_channel != "canary": + continue + latest_canary_version = version + latest_canary_commit = previous_commit + break + + if not latest_canary_version: + reason = "No existing canary found on TestPyPI." + if ignored_release_count: + reason = f"{reason} Ignored {ignored_release_count} unreadable release(s) while searching." + return { + "should_publish": "true", + "reason": reason, + "previous_version": "", + "previous_commit_sha": "", + } + + previous_short = latest_canary_commit[:7] + if latest_canary_commit == current_commit: + return { + "should_publish": "false", + "reason": f"Current HEAD {current_short} is already published as canary {latest_canary_version} on TestPyPI.", + "previous_version": latest_canary_version, + "previous_commit_sha": previous_short, + } + + return { + "should_publish": "true", + "reason": f"Latest TestPyPI canary {latest_canary_version} is from {previous_short}, which does not match HEAD {current_short}.", + "previous_version": latest_canary_version, + "previous_commit_sha": previous_short, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + prepare_parser = subparsers.add_parser("prepare") + prepare_subparsers = prepare_parser.add_subparsers(dest="release_type", required=True) + + prerelease_parser = prepare_subparsers.add_parser("prerelease") + prerelease_parser.add_argument("--run-number", type=int, required=True) + prerelease_parser.add_argument("--github-output", type=pathlib.Path) + + canary_parser = prepare_subparsers.add_parser("canary") + canary_parser.add_argument("target_ref") + canary_parser.add_argument("--run-number", type=int, required=True) + canary_parser.add_argument("--github-output", type=pathlib.Path) + + check_canary_parser = subparsers.add_parser("check-canary") + check_canary_parser.add_argument("--github-output", type=pathlib.Path) + + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + if args.command == "prepare": + target_ref = getattr(args, "target_ref", "") + outputs = prepare_release(args.release_type, target_ref, args.run_number) + elif args.command == "check-canary": + outputs = check_canary_status() + else: # pragma: no cover + raise ValueError(f"Unsupported command '{args.command}'") + + github_output = getattr(args, "github_output", None) + if github_output is not None: + write_github_output(github_output, outputs) + else: + for key, value in outputs.items(): + print(f"{key}={value}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/py/src/braintrust/test_version.py b/py/src/braintrust/test_version.py index 1324cbe8..27a5a520 100644 --- a/py/src/braintrust/test_version.py +++ b/py/src/braintrust/test_version.py @@ -14,10 +14,13 @@ def test_version(): # Basic assertions that should always pass assert version.VERSION assert version.GIT_COMMIT + assert version.RELEASE_CHANNEL assert isinstance(version.VERSION, str) assert isinstance(version.GIT_COMMIT, str) + assert isinstance(version.RELEASE_CHANNEL, str) assert len(version.VERSION) > 0 assert len(version.GIT_COMMIT) > 0 + assert len(version.RELEASE_CHANNEL) > 0 if is_from_wheel: # When testing from the wheel, GIT_COMMIT # should be the actual commit hash, not the placeholder @@ -25,3 +28,4 @@ def test_version(): else: # When testing from source directly, we expect to see the placeholder assert version.GIT_COMMIT == "__GIT_COMMIT__" + assert version.RELEASE_CHANNEL == "source" diff --git a/py/src/braintrust/version.py b/py/src/braintrust/version.py index fb7f65a8..ef3ab4c8 100644 --- a/py/src/braintrust/version.py +++ b/py/src/braintrust/version.py @@ -1,4 +1,5 @@ VERSION = "0.9.0" +RELEASE_CHANNEL = "source" # this will be templated during the build GIT_COMMIT = "__GIT_COMMIT__"