openclaw-performance.yml

name: OpenClaw Performance

run-name: ${{ inputs.dispatch_id != '' && format('OpenClaw Performance {0}', inputs.dispatch_id) || 'OpenClaw Performance' }}

on:
  schedule:
    - cron: "11 5 * * *"
  workflow_dispatch:
    inputs:
      target_ref:
        description: OpenClaw ref to benchmark; defaults to the workflow ref
        required: false
        default: ""
        type: string
      profile:
        description: Kova profile to run
        required: false
        default: diagnostic
        type: choice
        options:
          - smoke
          - diagnostic
          - soak
          - release
      repeat:
        description: Repeat count for non-profiled Kova runs
        required: false
        default: "3"
        type: string
      deep_profile:
        description: Run the deep-profile lane with CPU/heap/trace artifacts
        required: false
        default: false
        type: boolean
      live_openai_candidate:
        description: Run the live OpenAI GPT 5.6 agent-turn lane
        required: false
        default: false
        type: boolean
      fail_on_regression:
        description: Fail the workflow when Kova exits non-zero
        required: false
        default: false
        type: boolean
      publish_reports:
        description: Publish completed reports to openclaw/clawgrit-reports
        required: false
        default: true
        type: boolean
      kova_ref:
        description: Optional openclaw/Kova Git ref override; defaults to the fixture compatible with the target config schema
        required: false
        default: ""
        type: string
      kova_config_contract:
        description: Optional fixture-contract override for a custom Kova ref
        required: false
        default: ""
        type: string
      dispatch_id:
        description: Optional parent workflow dispatch identifier
        required: false
        default: ""
        type: string

permissions:
  contents: read

concurrency:
  group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-{1}', github.workflow, github.run_id) || format('{0}-{1}', github.workflow, github.ref) }}
  cancel-in-progress: false

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
  OCM_VERSION: v0.2.32
  OCM_LINUX_X64_SHA256: 5b20c21b2825f69b89eb37baa657f0f0062124517e6e6828e9857c7e9bbd3070
  KOVA_REPOSITORY: openclaw/Kova
  KOVA_CANONICAL_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78
  KOVA_LEGACY_LIST_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78
  PERFORMANCE_MODEL_ID: gpt-5.6-luna
  # Release matrices cold-build the candidate runtime before measurement.
  KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }}

jobs:
  resolve_target:
    name: Resolve target ref
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    outputs:
      checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
      tested_ref: ${{ steps.resolve.outputs.tested_ref }}
      tested_sha: ${{ steps.resolve.outputs.tested_sha }}
      kova_ref: ${{ steps.resolve.outputs.kova_ref }}
      kova_config_contract: ${{ steps.resolve.outputs.kova_config_contract }}
      kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }}
    steps:
      - name: Resolve OpenClaw target ref
        id: resolve
        env:
          GH_TOKEN: ${{ github.token }}
          TARGET_REF_INPUT: ${{ inputs.target_ref }}
          KOVA_REF_INPUT: ${{ inputs.kova_ref }}
          KOVA_CONFIG_CONTRACT_INPUT: ${{ inputs.kova_config_contract }}
        shell: bash
        run: |
          set -euo pipefail
          requested="${TARGET_REF_INPUT:-}"
          if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then
            echo "::error::target_ref must be a single line."
            exit 1
          fi
          if [[ -z "$requested" ]]; then
            resolved_sha="$GITHUB_SHA"
            tested_ref="$GITHUB_REF_NAME"
          else
            encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")"
            if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then
              echo "::error::Unable to resolve OpenClaw target_ref '${requested}'."
              exit 1
            fi
            tested_ref="$requested"
          fi
          if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then
            echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'."
            exit 1
          fi

          kova_ref="${KOVA_REF_INPUT:-}"
          kova_config_contract="${KOVA_CONFIG_CONTRACT_INPUT:-}"
          if [[ "$kova_ref" == *$'\n'* || "$kova_ref" == *$'\r'* ]]; then
            echo "::error::kova_ref must be a single line."
            exit 1
          fi
          if [[ "$kova_config_contract" == *$'\n'* || "$kova_config_contract" == *$'\r'* ]]; then
            echo "::error::kova_config_contract must be a single line."
            exit 1
          fi

          if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then
            if schema_content="$({
              gh api "repos/${GITHUB_REPOSITORY}/contents/src/config/zod-schema.agent-defaults.ts?ref=${resolved_sha}" --jq '.content' |
                base64 --decode
            })"; then
              :
            elif [[ -z "$kova_ref" ]]; then
              echo "::error::Unable to inspect the Kova config-fixture contract for target ${resolved_sha}. Supply kova_ref explicitly and optionally set kova_config_contract for that producer."
              exit 1
            else
              schema_content=""
            fi
            if [[ -n "$schema_content" ]] && grep -Fqx '    mediaModels: z' <<<"$schema_content"; then
              detected_kova_config_contract="canonical"
              default_kova_ref="$KOVA_CANONICAL_CONFIG_REF"
            elif [[ -n "$schema_content" ]] && grep -Fqx '    imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then
              detected_kova_config_contract="legacy-list"
              default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF"
            else
              detected_kova_config_contract=""
              default_kova_ref=""
            fi
            if [[ -n "$detected_kova_config_contract" ]]; then
              kova_config_contract="${kova_config_contract:-$detected_kova_config_contract}"
              kova_ref="${kova_ref:-$default_kova_ref}"
            elif [[ -z "$kova_ref" ]]; then
              echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly and optionally set kova_config_contract for that producer."
              exit 1
            fi
          fi

          echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT"
          echo "tested_ref=$tested_ref" >> "$GITHUB_OUTPUT"
          echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT"
          echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT"
          echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"
          if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then
            echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"
          else
            echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT"
          fi

  kova:
    name: ${{ matrix.title }}
    needs: resolve_target
    runs-on: blacksmith-16vcpu-ubuntu-2404
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix:
        include:
          - lane: mock-provider
            title: Kova mock provider performance
            auth: mock
            repeat: input
            deep_profile: "false"
            live: "false"
            managed_service: "true"
            include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:agent-cold-warm-message"
            expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,bundled-plugin-startup:fresh,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
          - lane: mock-deep-profile
            title: Kova mock provider deep profile
            auth: mock
            repeat: "1"
            deep_profile: "true"
            live: "false"
            managed_service: "true"
            include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message"
            expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
          - lane: live-openai-candidate
            title: Kova live OpenAI GPT 5.6 agent turn
            auth: live
            repeat: "1"
            deep_profile: "false"
            live: "true"
            managed_service: "false"
            include_filters: "scenario:agent-cold-warm-message"
            expected_release_entries: "agent-cold-warm-message:mock-openai-provider"
    env:
      KOVA_REF: ${{ needs.resolve_target.outputs.kova_ref }}
      KOVA_OPENCLAW_CONFIG_CONTRACT: ${{ needs.resolve_target.outputs.kova_config_contract }}
      KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }}
      PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
      REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }}
      BUNDLE_DIR: ${{ github.workspace }}/.artifacts/kova/bundles/${{ matrix.lane }}
      SUMMARY_DIR: ${{ github.workspace }}/.artifacts/kova/summaries
      LANE_ID: ${{ matrix.lane }}
      TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
      EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
      PROFILE: ${{ inputs.profile || 'diagnostic' }}
      REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
      FAIL_ON_REGRESSION: ${{ inputs.fail_on_regression || 'false' }}
      INCLUDE_FILTERS: ${{ matrix.include_filters }}
      EXPECTED_RELEASE_ENTRIES: ${{ matrix.expected_release_entries }}
      AUTH_MODE: ${{ matrix.auth }}
      MATRIX_REPEAT: ${{ matrix.repeat }}
      MATRIX_DEEP_PROFILE: ${{ matrix.deep_profile }}
      MATRIX_LIVE: ${{ matrix.live }}
      KOVA_REF_TRUSTED_FOR_LIVE: ${{ needs.resolve_target.outputs.kova_ref_trusted_for_live }}
    steps:
      - name: Decide lane
        id: lane
        shell: bash
        run: |
          set -euo pipefail
          run_lane=true
          reason=""
          if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
            run_lane=false
            reason="deep_profile input is false"
          fi
          if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
            run_lane=false
            reason="live_openai_candidate input is false"
          fi
          if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then
            echo "::error::The live OpenAI lane only executes a reviewed immutable Kova default. Omit kova_ref or update the pinned workflow defaults after review."
            exit 1
          fi
          echo "run=$run_lane" >> "$GITHUB_OUTPUT"
          if [[ "$run_lane" != "true" ]]; then
            echo "Skipping ${LANE_ID}: ${reason}" >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Checkout OpenClaw
        if: steps.lane.outputs.run == 'true'
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ needs.resolve_target.outputs.checkout_ref }}
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout performance workflow helpers
        if: steps.lane.outputs.run == 'true'
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ github.sha }}
          path: .artifacts/performance-workflow
          fetch-depth: 1
          persist-credentials: false

      - name: Record tested revision
        if: steps.lane.outputs.run == 'true'
        shell: bash
        run: |
          set -euo pipefail
          tested_sha="$(git rev-parse HEAD)"
          if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
            echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
            exit 1
          fi
          echo "TESTED_REF=${TARGET_REF}" >> "$GITHUB_ENV"
          echo "TESTED_SHA=${EXPECTED_TESTED_SHA}" >> "$GITHUB_ENV"
          {
            echo "Tested ref: ${TARGET_REF}"
            echo "Tested SHA: ${EXPECTED_TESTED_SHA}"
            echo "Workflow ref: ${GITHUB_REF_NAME}"
            echo "Workflow SHA: ${GITHUB_SHA}"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Set up Node environment
        if: steps.lane.outputs.run == 'true'
        uses: ./.github/actions/setup-node-env
        with:
          install-bun: "false"

      - name: Prepare systemd user session
        if: ${{ steps.lane.outputs.run == 'true' && matrix.managed_service == 'true' }}
        shell: bash
        run: |
          set -euo pipefail
          user="$(id -un)"
          uid="$(id -u)"

          test "$(ps -p 1 -o comm= | xargs)" = systemd
          sudo systemctl is-active --quiet systemd-logind.service
          sudo loginctl enable-linger "$user"
          sudo systemctl start "user@${uid}.service"

          runtime_dir="$(loginctl show-user "$user" --property=RuntimePath --value)"
          test -n "$runtime_dir"
          test -d "$runtime_dir"
          test -O "$runtime_dir"

          export XDG_RUNTIME_DIR="$runtime_dir"
          test -S "$XDG_RUNTIME_DIR/systemd/private"
          echo "XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR" >> "$GITHUB_ENV"

          if [[ -S "$runtime_dir/bus" ]]; then
            export DBUS_SESSION_BUS_ADDRESS="unix:path=${runtime_dir}/bus"
            echo "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" >> "$GITHUB_ENV"
          fi

          systemctl --user show-environment >/dev/null

      - name: Install OCM and Kova
        if: steps.lane.outputs.run == 'true'
        shell: bash
        run: |
          set -euo pipefail
          KOVA_SRC="${RUNNER_TEMP}/kova-src"
          echo "KOVA_SRC=$KOVA_SRC" >> "$GITHUB_ENV"
          mkdir -p "$HOME/.local/bin" "$(dirname "$KOVA_SRC")" "${RUNNER_TEMP}/ocm-install"

          ocm_archive="${RUNNER_TEMP}/ocm-${OCM_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
          # retry-max-time only gates new retries; cap an active transfer separately
          # so a stalled release asset cannot consume the entire Kova job deadline.
          curl -fsSL --proto '=https' --tlsv1.2 \
            --max-time 180 \
            --retry 8 --retry-max-time 180 --retry-all-errors --retry-connrefused \
            -o "$ocm_archive" \
            "https://github.com/shakkernerd/ocm/releases/download/${OCM_VERSION}/ocm-x86_64-unknown-linux-gnu.tar.gz"
          echo "${OCM_LINUX_X64_SHA256}  ${ocm_archive}" | sha256sum -c -
          tar -xzf "$ocm_archive" -C "${RUNNER_TEMP}/ocm-install"
          install -m 0755 "${RUNNER_TEMP}/ocm-install/ocm" "$HOME/.local/bin/ocm"

          git init -b main "$KOVA_SRC"
          git -C "$KOVA_SRC" remote add origin "https://github.com/${KOVA_REPOSITORY}.git"
          git -C "$KOVA_SRC" fetch --filter=blob:none --depth 1 origin "$KOVA_REF"
          git -C "$KOVA_SRC" checkout --detach FETCH_HEAD
          npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund
          node - "$KOVA_SRC" <<'NODE'
          const fs = require("node:fs");
          const path = require("node:path");

          const root = process.argv[2];
          const packageJsonPath = require.resolve("mock-ai-provider/package.json", {
            paths: [root],
          });
          const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
          const bin =
            typeof packageJson.bin === "string"
              ? packageJson.bin
              : packageJson.bin?.["mock-ai-provider"];
          if (typeof bin !== "string" || bin.length === 0) {
            throw new Error("mock-ai-provider does not declare its CLI entrypoint");
          }
          fs.accessSync(path.resolve(path.dirname(packageJsonPath), bin), fs.constants.R_OK);
          fs.accessSync(
            path.join(root, "node_modules", ".bin", "mock-ai-provider"),
            fs.constants.X_OK,
          );
          require.resolve("zod", { paths: [root] });
          NODE
          cat > "$HOME/.local/bin/kova" <<EOF
          #!/usr/bin/env bash
          export KOVA_HOME="${KOVA_HOME}"
          exec node "${KOVA_SRC}/bin/kova.mjs" "\$@"
          EOF
          chmod 0755 "$HOME/.local/bin/kova"
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Kova version and plan sanity
        if: steps.lane.outputs.run == 'true'
        shell: bash
        run: |
          set -euo pipefail
          repeat="$REQUESTED_REPEAT"
          if [[ "$MATRIX_REPEAT" != "input" ]]; then
            repeat="$MATRIX_REPEAT"
          fi
          plan_dir="${RUNNER_TEMP}/kova-plans"
          plan_json="${plan_dir}/${LANE_ID}.json"
          mkdir -p "$plan_dir"

          kova version --json
          kova matrix plan \
            --profile "$PROFILE" \
            --target "local-build:${GITHUB_WORKSPACE}" \
            --include "$INCLUDE_FILTERS" \
            --parallel 1 \
            --repeat "$repeat" \
            --json >"$plan_json"
          node --input-type=module - "$plan_json" <<'NODE'
          import fs from "node:fs";

          const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
          const expectedFilters = process.env.INCLUDE_FILTERS.split(",");
          const actualFilters = plan.controls?.include;
          if (
            !Array.isArray(actualFilters) ||
            actualFilters.length !== expectedFilters.length ||
            actualFilters.some((filter, index) => filter !== expectedFilters[index])
          ) {
            throw new Error("Kova plan did not preserve the requested include filters");
          }

          if (process.env.PROFILE === "release") {
            const expectedEntries = process.env.EXPECTED_RELEASE_ENTRIES.split(",").sort();
            if (!Array.isArray(plan.entries)) {
              throw new Error("Kova release plan did not contain entries");
            }
            const actualEntries = plan.entries
              .map((entry) => {
                if (entry.status !== "SELECTED" || !entry.scenario?.id || !entry.state?.id) {
                  throw new Error("Kova release plan contained an invalid selected entry");
                }
                return `${entry.scenario.id}:${entry.state.id}`;
              })
              .sort();
            if (
              actualEntries.length !== expectedEntries.length ||
              actualEntries.some((entry, index) => entry !== expectedEntries[index])
            ) {
              throw new Error("Kova release plan entries did not match the required lane coverage");
            }
          }
          NODE
          echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"
          echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV"

      - name: Configure live OpenAI auth
        if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
        shell: bash
        run: |
          set -euo pipefail
          if [[ -z "${OPENAI_API_KEY:-}" ]]; then
            echo "OPENAI_API_KEY is not configured; live GPT 5.6 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY"
            exit 1
          fi
          kova setup --ci --json
          kova setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY --json

      - name: Run Kova
        id: kova
        if: steps.lane.outputs.run == 'true'
        env:
          OPENAI_API_KEY: ${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}
          OPENAI_BASE_URL: ${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR"

          repeat="$KOVA_LANE_REPEAT"

          args=(
            matrix run
            --profile "$PROFILE"
            --target "local-build:${GITHUB_WORKSPACE}"
            --include "$INCLUDE_FILTERS"
            --auth "$AUTH_MODE"
            --parallel 1
            --repeat "$repeat"
            --timeout-ms "$KOVA_SCENARIO_TIMEOUT_MS"
            --report-dir "$REPORT_DIR"
            --execute
            --json
          )

          if [[ "$AUTH_MODE" == "live" ]]; then
            args+=(--model "$PERFORMANCE_MODEL_ID")
          fi
          if [[ "$MATRIX_DEEP_PROFILE" == "true" ]]; then
            args+=(--deep-profile)
          fi
          if [[ "$FAIL_ON_REGRESSION" == "true" ]]; then
            args+=(--gate)
          fi

          log_path="$REPORT_DIR/${LANE_ID}.log"
          set +e
          kova "${args[@]}" 2>&1 | tee "$log_path"
          status=${PIPESTATUS[0]}
          set -e

          report_json="$(node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR")"
          report_md="${report_json%.json}.md"
          echo "status=$status" >> "$GITHUB_OUTPUT"
          echo "report_json=$report_json" >> "$GITHUB_OUTPUT"
          echo "report_md=$report_md" >> "$GITHUB_OUTPUT"

          set +e
          node --import tsx "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mts" \
            --plan "$KOVA_PLAN_JSON" \
            --report "$report_json" \
            --profile "$PROFILE" \
            --target "local-build:${GITHUB_WORKSPACE}" \
            --repeat "$repeat" \
            --include "$INCLUDE_FILTERS" \
            --auth "$AUTH_MODE" \
            --model "$PERFORMANCE_MODEL_ID"
          evidence_status=$?
          set -e

          effective_status="$status"
          if [[ "$evidence_status" == "0" && "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
            gate_args=("$report_json")
            if [[ "$KOVA_REF" == "$KOVA_CANONICAL_CONFIG_REF" || "$KOVA_REF" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then
              gate_args+=(--require-instrumented-performance-contract)
            fi
            if node --import tsx "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mts" "${gate_args[@]}"
            then
              effective_status=0
              {
                echo "Kova returned a non-zero release-gate verdict, but the trusted report adapter found only filtered coverage or profiling-affected resource thresholds with no baseline regression."
                echo
              } >> "$GITHUB_STEP_SUMMARY"
            fi
          fi
          echo "effective_status=$effective_status" >> "$GITHUB_OUTPUT"

          set +e
          kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json"
          bundle_status=${PIPESTATUS[0]}
          set -e

          summary_path="$SUMMARY_DIR/${LANE_ID}.md"
          summary_args=(node --import tsx "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mts" --report "$report_json" --output "$summary_path" --lane "$LANE_ID")
          set +e
          "${summary_args[@]}"
          summary_status=$?
          set -e
          if [[ "$summary_status" != "0" ]]; then
            cat > "$summary_path" <<EOF
          # OpenClaw Performance Report

          - Lane: ${LANE_ID}
          - Kova report: ${report_json}
          - Summary generation failed with status ${summary_status}; inspect the Kova report and lane log.
          EOF
          fi
          cat >> "$summary_path" <<EOF

          ## Test scope

          - Repository: ${GITHUB_REPOSITORY}
          - Tested ref: ${TESTED_REF}
          - Tested SHA: ${TESTED_SHA}
          - Workflow ref: ${GITHUB_REF_NAME}
          - Workflow SHA: ${GITHUB_SHA}
          - Kova repository: ${KOVA_REPOSITORY}
          - Kova ref: ${KOVA_REF}
          - Kova profile: ${PROFILE}
          - Kova scenario timeout: ${KOVA_SCENARIO_TIMEOUT_MS}ms
          - Lane auth: ${AUTH_MODE}
          - Lane model: ${PERFORMANCE_MODEL_ID}
          - Lane repeat: ${repeat}
          - Include filters: ${INCLUDE_FILTERS}
          EOF
          cat "$summary_path" >> "$GITHUB_STEP_SUMMARY"

          if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]; then
            exit 1
          fi
          if [[ "$FAIL_ON_REGRESSION" == "true" && "$effective_status" != "0" ]]; then
            exit "$effective_status"
          fi

      - name: Validate Kova evidence
        if: ${{ always() && steps.lane.outputs.run == 'true' }}
        shell: bash
        run: |
          set -euo pipefail
          missing=0
          if ! node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR" >/dev/null; then
            echo "::error::Exactly one full Kova JSON report is required for ${LANE_ID}."
            missing=1
          fi
          if [[ ! -s "$BUNDLE_DIR/bundle.json" ]]; then
            echo "::error::Kova bundle evidence is missing for ${LANE_ID}."
            missing=1
          fi
          if [[ ! -s "$SUMMARY_DIR/${LANE_ID}.md" ]]; then
            echo "::error::Kova summary evidence is missing for ${LANE_ID}."
            missing=1
          fi
          if [[ "$missing" != "0" ]]; then
            exit 1
          fi

      - name: Upload Kova artifacts
        if: ${{ always() && steps.lane.outputs.run == 'true' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: openclaw-performance-${{ matrix.lane }}-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            .artifacts/kova/reports/${{ matrix.lane }}
            .artifacts/kova/bundles/${{ matrix.lane }}
            .artifacts/kova/summaries/${{ matrix.lane }}.md
          if-no-files-found: error
          retention-days: ${{ matrix.deep_profile == 'true' && 14 || 30 }}

  source_performance:
    name: OpenClaw source performance probes
    needs: resolve_target
    runs-on: blacksmith-16vcpu-ubuntu-2404
    timeout-minutes: 120
    env:
      PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
      SOURCE_PERF_DIR: ${{ github.workspace }}/.artifacts/openclaw-performance/source/mock-provider
      TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
      EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
      TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
      TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
      REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
    steps:
      - name: Checkout OpenClaw source target
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ needs.resolve_target.outputs.checkout_ref }}
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout source performance helpers
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ github.sha }}
          path: .artifacts/performance-workflow
          fetch-depth: 1
          persist-credentials: false

      - name: Record source performance revision
        shell: bash
        run: |
          set -euo pipefail
          tested_sha="$(git rev-parse HEAD)"
          if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
            echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
            exit 1
          fi
          {
            echo "Tested ref: ${TESTED_REF}"
            echo "Tested SHA: ${TESTED_SHA}"
            echo "Workflow ref: ${GITHUB_REF_NAME}"
            echo "Workflow SHA: ${GITHUB_SHA}"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Set up source performance environment
        uses: ./.github/actions/setup-node-env
        with:
          install-bun: "false"

      - name: Fetch previous source performance baseline
        shell: bash
        run: |
          set -euo pipefail
          reports_root=".artifacts/clawgrit-baseline"
          mkdir -p "$reports_root"
          git -C "$reports_root" init -b main
          git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
          if ! git -C "$reports_root" fetch --filter=blob:none --depth=1 origin main; then
            echo "No previous source performance baseline could be fetched." >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')"
          pointer="openclaw-performance/${ref_slug}/latest-mock-provider.json"
          if ! git -C "$reports_root" cat-file -e "FETCH_HEAD:${pointer}"; then
            echo "No previous source performance baseline exists for ${TESTED_REF}." >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          if ! latest_path="$(git -C "$reports_root" show "FETCH_HEAD:${pointer}" | node -e "const fs=require('node:fs'); const data=JSON.parse(fs.readFileSync(0,'utf8')); const value=String(data.path || ''); if (!/^openclaw-performance\\/[A-Za-z0-9._-]+\\/[0-9]+-[0-9]+\\/mock-provider$/u.test(value)) process.exit(1); process.stdout.write(value);")"; then
            echo "Previous source performance baseline pointer is invalid." >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          git -C "$reports_root" sparse-checkout init --no-cone
          printf '/%s/source/\n' "$latest_path" | git -C "$reports_root" sparse-checkout set --stdin
          git -C "$reports_root" checkout --detach FETCH_HEAD
          baseline_source="${reports_root}/${latest_path}/source"
          if [[ -d "$baseline_source" ]]; then
            baseline_source="$(realpath "$baseline_source")"
            echo "SOURCE_PERF_BASELINE_DIR=$baseline_source" >> "$GITHUB_ENV"
            echo "Using source performance baseline: ${latest_path}/source" >> "$GITHUB_STEP_SUMMARY"
          else
            echo "Previous source performance baseline has no source directory." >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Run OpenClaw source performance probes
        shell: bash
        run: |
          set -euo pipefail
          source_runs="$REQUESTED_REPEAT"
          if ! [[ "$source_runs" =~ ^[0-9]+$ ]] || [[ "$source_runs" -lt 1 ]]; then
            source_runs=3
          fi

          mkdir -p "$SOURCE_PERF_DIR/mock-hello"
          if ! node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:gateway:cpu-scenarios'] && scripts['test:extensions:memory'] && scripts.openclaw && fs.existsSync('openclaw.mjs') && fs.existsSync('scripts/profile-extension-memory.mts') ? 0 : 1)"; then
            cat > "$SOURCE_PERF_DIR/index.md" <<EOF
          # OpenClaw Source Performance

          Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)

          Source probes skipped for this tested ref because one or more probe entry points are not present in the checked-out source tree.

          ## Test scope

          - Tested ref: ${TESTED_REF}
          - Tested SHA: ${TESTED_SHA}
          - Required scripts: test:gateway:cpu-scenarios, test:extensions:memory, openclaw, openclaw.mjs, scripts/profile-extension-memory.mts
          EOF
            cat "$SOURCE_PERF_DIR/index.md" >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi

          # target_ref may predate the dedicated profile; preserve the historical full build there.
          if node --import tsx -e "import('./scripts/build-all.mts').then((module) => process.exit(module.BUILD_ALL_PROFILES?.sourcePerformance ? 0 : 1)).catch(() => process.exit(1))"; then
            OPENCLAW_BUILD_PRIVATE_QA=1 node --import tsx scripts/build-all.mts sourcePerformance
          else
            pnpm build
          fi

          supported_startup_cases="$(
            node --import tsx scripts/bench-gateway-startup.ts --help |
              sed -n 's/^  \([[:alnum:]_-][[:alnum:]_-]*\) (.*/\1/p'
          )"
          startup_case_args=()
          for startup_case in default skipChannels preparedRuntimeCatalogStall preparedRuntimeScaleOne preparedRuntimeScaleMany oneInternalHook allInternalHooks fiftyPlugins fiftyStartupLazyPlugins; do
            if grep -Fxq "$startup_case" <<<"$supported_startup_cases"; then
              startup_case_args+=(--startup-case "$startup_case")
            fi
          done
          if [[ " ${startup_case_args[*]} " != *" --startup-case default "* ]]; then
            echo "The target startup benchmark did not advertise its required default case." >&2
            exit 1
          fi

          pnpm test:gateway:cpu-scenarios \
            --output-dir "$SOURCE_PERF_DIR/gateway-cpu" \
            --runs "$source_runs" \
            --warmup 1 \
            --skip-qa \
            "${startup_case_args[@]}"

          pnpm test:extensions:memory \
            -- --json "$SOURCE_PERF_DIR/extension-memory.json"

          for run_index in $(seq 1 "$source_runs"); do
            run_dir="$SOURCE_PERF_DIR/mock-hello/run-$(printf '%03d' "$run_index")"
            pnpm openclaw qa suite \
              --provider-mode mock-openai \
              --model "mock-openai/${PERFORMANCE_MODEL_ID}" \
              --concurrency 1 \
              --output-dir "$(realpath --relative-to="$GITHUB_WORKSPACE" "$run_dir")" \
              --scenario channel-chat-baseline
          done

          gateway_home="$(mktemp -d)"
          gateway_readiness_home="$(mktemp -d)"
          gateway_port="$(node -e "const net=require('node:net'); const s=net.createServer(); s.listen(0,'127.0.0.1',()=>{ console.log(s.address().port); s.close(); });")"
          gateway_token="$(node -e "console.log(require('node:crypto').randomBytes(32).toString('hex'))")"
          gateway_state="$gateway_home/.openclaw"
          gateway_config="$gateway_state/openclaw.json"
          gateway_readiness_state="$gateway_readiness_home/.openclaw"
          gateway_readiness_config="$gateway_readiness_state/openclaw.json"
          gateway_log="$SOURCE_PERF_DIR/cli-gateway.log"
          gateway_readiness_log="$SOURCE_PERF_DIR/cli-gateway-readiness.log"
          gateway_pid=""
          mkdir -p "$gateway_state" "$gateway_readiness_state"
          catalog_refresh_config=""
          # Frozen release candidates can predate this config key. Inspect the
          # target schema so the trusted harness never writes invalid config.
          if rg -q 'catalogRefresh:' src/config/zod-schema.core.ts; then
            catalog_refresh_config='    "models": { "catalogRefresh": { "enabled": false } },'
          fi
          cat > "$gateway_config" <<EOF
          {
            "agents": { "defaults": { "heartbeat": { "every": "0m" } } },
            "browser": { "enabled": false },
          ${catalog_refresh_config}
            "update": { "checkOnStart": false },
            "gateway": {
              "mode": "local",
              "port": ${gateway_port},
              "bind": "loopback",
              "auth": { "mode": "token" },
              "controlUi": { "enabled": false },
              "tailscale": { "mode": "off" }
            },
            "plugins": {
              "enabled": true,
              "entries": { "browser": { "enabled": false } }
            }
          }
          EOF
          cp "$gateway_config" "$gateway_readiness_config"
          : > "$gateway_readiness_log"
          cleanup_gateway() {
            if [[ -n "${gateway_pid:-}" ]] && kill -0 "$gateway_pid" 2>/dev/null; then
              kill "$gateway_pid" 2>/dev/null || true
              wait "$gateway_pid" 2>/dev/null || true
            fi
            rm -rf "$gateway_home" "$gateway_readiness_home"
          }
          trap cleanup_gateway EXIT
          OPENCLAW_GATEWAY_TOKEN="$gateway_token" OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" OPENCLAW_SKIP_CHANNELS=1 OPENCLAW_SKIP_CRON=1 \
            node dist/entry.js gateway run --bind loopback --port "$gateway_port" --auth token --allow-unconfigured --force \
            >"$gateway_log" 2>&1 &
          gateway_pid="$!"

          gateway_ready_timeout_seconds=120
          gateway_probe_timeout_seconds=5
          gateway_ready_deadline=$((SECONDS + gateway_ready_timeout_seconds))
          while true; do
            gateway_ready_remaining=$((gateway_ready_deadline - SECONDS))
            if (( gateway_ready_remaining <= 0 )); then
              cat "$gateway_log" >&2
              echo "Timed out after ${gateway_ready_timeout_seconds}s waiting for gateway health." >&2
              exit 1
            fi
            # Clamp each probe to the remaining startup budget so a stalled response cannot multiply the wait.
            gateway_probe_timeout="$gateway_ready_remaining"
            if (( gateway_probe_timeout > gateway_probe_timeout_seconds )); then
              gateway_probe_timeout="$gateway_probe_timeout_seconds"
            fi
            if curl -fsS --connect-timeout 2 --max-time "$gateway_probe_timeout" "http://127.0.0.1:${gateway_port}/healthz" >/dev/null; then
              break
            fi
            if ! kill -0 "$gateway_pid" 2>/dev/null; then
              cat "$gateway_log" >&2
              exit 1
            fi
            sleep 1
          done

          # Exercise the same shared-secret WebSocket path without creating benchmark device state.
          while true; do
            gateway_ready_remaining=$((gateway_ready_deadline - SECONDS))
            if (( gateway_ready_remaining <= 0 )); then
              cat "$gateway_log" >&2
              cat "$gateway_readiness_log" >&2
              echo "Timed out after ${gateway_ready_timeout_seconds}s waiting for gateway WebSocket health." >&2
              exit 1
            fi
            gateway_ready_remaining_ms=$((gateway_ready_remaining * 1000))
            if OPENCLAW_GATEWAY_TOKEN="$gateway_token" OPENCLAW_HOME="$gateway_readiness_home" OPENCLAW_STATE_DIR="$gateway_readiness_state" OPENCLAW_CONFIG_PATH="$gateway_readiness_config" \
              node dist/entry.js gateway health \
                --port "$gateway_port" \
                --timeout "$gateway_ready_remaining_ms" \
                --json >"$gateway_readiness_log" 2>&1; then
              break
            fi
            if ! kill -0 "$gateway_pid" 2>/dev/null; then
              cat "$gateway_log" >&2
              cat "$gateway_readiness_log" >&2
              exit 1
            fi
            # Pace fast transport failures without extending the shared startup deadline.
            gateway_ready_remaining=$((gateway_ready_deadline - SECONDS))
            if (( gateway_ready_remaining > 0 )); then
              sleep 1
            fi
          done

          OPENCLAW_GATEWAY_TOKEN="$gateway_token" OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" \
            node --import tsx "$PERFORMANCE_HELPER_DIR/scripts/bench-cli-startup.ts" \
            --entry "$GITHUB_WORKSPACE/openclaw.mjs" \
            --case gatewayHealthJsonWarmState \
            --case gatewayHealthJsonFreshState \
            --case configGetGatewayPort \
            --runs "$source_runs" \
            --warmup 1 \
            --output "$SOURCE_PERF_DIR/cli-startup.json"
          cleanup_gateway
          trap - EXIT

          if node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:sqlite:perf:smoke'] && fs.existsSync('scripts/bench-sqlite-state.ts') ? 0 : 1)"; then
            pnpm test:sqlite:perf:smoke
            cp .artifacts/sqlite-perf/smoke.json "$SOURCE_PERF_DIR/sqlite-perf-smoke.json"
          else
            echo "SQLite state smoke probe is not available in ${TESTED_REF}; continuing with the remaining source probes." >> "$GITHUB_STEP_SUMMARY"
          fi

          summary_args=(node --import tsx "$PERFORMANCE_HELPER_DIR/scripts/openclaw-performance-source-summary.mts" \
            --source-dir "$SOURCE_PERF_DIR" \
            --output "$SOURCE_PERF_DIR/index.md")
          if [[ -n "${SOURCE_PERF_BASELINE_DIR:-}" && -d "$SOURCE_PERF_BASELINE_DIR" ]]; then
            summary_args+=(--baseline-source-dir "$SOURCE_PERF_BASELINE_DIR")
          fi
          "${summary_args[@]}"

          cat "$SOURCE_PERF_DIR/index.md" >> "$GITHUB_STEP_SUMMARY"

      - name: Upload source performance artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: openclaw-performance-source-${{ github.run_id }}-${{ github.run_attempt }}
          path: .artifacts/openclaw-performance/source
          if-no-files-found: error
          retention-days: 30

  publish:
    name: Publish ${{ matrix.title }} report
    needs: [resolve_target, kova, source_performance]
    if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    permissions:
      actions: read
      contents: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - lane: mock-provider
            title: mock provider
          - lane: mock-deep-profile
            title: mock deep profile
          - lane: live-openai-candidate
            title: live OpenAI candidate
    env:
      LANE_ID: ${{ matrix.lane }}
      PERFORMANCE_PUBLISHER_HELPER: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-publish-files.mjs
      PERFORMANCE_REPORT_SELECTOR: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-selector.mjs
      PUBLISHED_REPORT_MAX_FILE_BYTES: "50000000"
      REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }}
    steps:
      - name: Decide report publication lane
        id: lane
        shell: bash
        run: |
          set -euo pipefail
          run_lane=true
          if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
            run_lane=false
          fi
          if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
            run_lane=false
          fi
          echo "run=$run_lane" >> "$GITHUB_OUTPUT"

      - name: Checkout performance publisher helper
        if: steps.lane.outputs.run == 'true'
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ github.sha }}
          path: .artifacts/performance-publisher
          sparse-checkout: |
            scripts/lib/kova-report-publish-files.mjs
            scripts/lib/kova-report-selector.mjs
          sparse-checkout-cone-mode: false
          fetch-depth: 1
          persist-credentials: false

      - name: Resolve Kova artifact
        id: artifact
        if: steps.lane.outputs.run == 'true'
        continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
        env:
          GH_TOKEN: ${{ github.token }}
        shell: bash
        run: |
          set -euo pipefail
          annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
          prefix="openclaw-performance-${LANE_ID}-${GITHUB_RUN_ID}-"
          source_prefix="openclaw-performance-source-${GITHUB_RUN_ID}-"
          if ! artifacts="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" --jq '.artifacts[] | select(.expired == false) | [.id, .name] | @tsv')"; then
            echo "::${annotation}::Unable to list Kova artifacts for this workflow run."
            exit 1
          fi

          artifact_id=""
          producer_attempt=0
          source_artifact_id=""
          source_producer_attempt=0
          while IFS=$'\t' read -r candidate_id candidate_name; do
            if [[ "$candidate_name" == "$prefix"* ]]; then
              candidate_attempt="${candidate_name#"$prefix"}"
              [[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
              if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > producer_attempt )); then
                artifact_id="$candidate_id"
                producer_attempt="$candidate_attempt"
              fi
            elif [[ "$LANE_ID" == "mock-provider" && "$candidate_name" == "$source_prefix"* ]]; then
              candidate_attempt="${candidate_name#"$source_prefix"}"
              [[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
              if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > source_producer_attempt )); then
                source_artifact_id="$candidate_id"
                source_producer_attempt="$candidate_attempt"
              fi
            fi
          done <<< "$artifacts"

          if [[ -z "$artifact_id" ]]; then
            echo "::${annotation}::No Kova artifact is available for ${LANE_ID} through attempt ${GITHUB_RUN_ATTEMPT}."
            exit 1
          fi
          artifact_ids="$artifact_id"
          if [[ "$LANE_ID" == "mock-provider" ]]; then
            if [[ -z "$source_artifact_id" ]]; then
              echo "::${annotation}::No source performance artifact is available through attempt ${GITHUB_RUN_ATTEMPT}."
              exit 1
            fi
            artifact_ids="${artifact_ids},${source_artifact_id}"
            if (( source_producer_attempt > producer_attempt )); then
              producer_attempt="$source_producer_attempt"
            fi
          fi
          echo "id=$artifact_id" >> "$GITHUB_OUTPUT"
          echo "ids=$artifact_ids" >> "$GITHUB_OUTPUT"
          echo "producer_attempt=$producer_attempt" >> "$GITHUB_OUTPUT"
          echo "source_producer_attempt=$source_producer_attempt" >> "$GITHUB_OUTPUT"

      - name: Create isolated publisher paths
        id: paths
        if: steps.artifact.outputs.id != ''
        shell: bash
        run: |
          set -euo pipefail
          input_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX")"
          reports_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX")"
          chmod 0700 "$input_root" "$reports_root"
          echo "input_root=$input_root" >> "$GITHUB_OUTPUT"
          echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"

      - name: Download Kova artifacts
        id: download
        if: steps.artifact.outputs.id != ''
        continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          artifact-ids: ${{ steps.artifact.outputs.ids }}
          path: ${{ steps.paths.outputs.input_root }}

      - name: Prepare clawgrit report commit
        id: prepare
        if: steps.download.outcome == 'success'
        continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
        env:
          ARTIFACT_ID: ${{ steps.artifact.outputs.id }}
          INPUT_ROOT: ${{ steps.paths.outputs.input_root }}
          REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }}
          TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
          TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
          PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.producer_attempt }}
          SOURCE_PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.source_producer_attempt }}
        shell: bash
        run: |
          set -euo pipefail
          echo "ready=false" >> "$GITHUB_OUTPUT"
          annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
          input_root="$(realpath "$INPUT_ROOT")"
          reports_root="$(realpath "$REPORTS_ROOT")"
          if [[ "$input_root" != "$RUNNER_TEMP/"* || "$reports_root" != "$RUNNER_TEMP/"* ]]; then
            echo "::${annotation}::Publisher paths escaped RUNNER_TEMP."
            exit 1
          fi
          if [[ ! "$ARTIFACT_ID" =~ ^[0-9]+$ || ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" || ! "$SOURCE_PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$SOURCE_PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then
            echo "::${annotation}::Kova artifact attempt metadata is invalid."
            exit 1
          fi

          # download-artifact extracts one ID directly, but nests multiple IDs
          # under artifact names. Recursive Kova lookup supports both layouts.
          mapfile -d '' report_dirs < <(find "$input_root" -type d -path "*/reports/${LANE_ID}" -print0)
          if [[ "${#report_dirs[@]}" != "1" ]]; then
            echo "::${annotation}::Expected exactly one Kova report directory for ${LANE_ID}; found ${#report_dirs[@]}."
            exit 1
          fi
          if ! report_json="$(node "$PERFORMANCE_REPORT_SELECTOR" --report-dir "${report_dirs[0]}")"; then
            echo "::${annotation}::Exactly one full Kova JSON report is required for ${LANE_ID}."
            exit 1
          fi
          report_json="$(realpath "$report_json")"
          if [[ "$report_json" != "$input_root/"* || ! -s "$report_json" ]]; then
            echo "::${annotation}::Kova JSON report is outside the downloaded artifact or empty."
            exit 1
          fi
          report_md_path="${report_json%.json}.md"
          if [[ -e "$report_md_path" ]]; then
            if [[ ! -f "$report_md_path" || -L "$report_md_path" ]]; then
              echo "::${annotation}::Kova Markdown report is not a canonical artifact file."
              exit 1
            fi
            report_md="$(realpath "$report_md_path")"
            if [[ "$report_md" != "$input_root/"* ]]; then
              echo "::${annotation}::Kova Markdown report escaped the downloaded artifact."
              exit 1
            fi
          else
            report_md=""
          fi

          mapfile -d '' summaries < <(find "$input_root" -type f -path "*/summaries/${LANE_ID}.md" -print0)
          mapfile -d '' bundles < <(find "$input_root" -type d -path "*/bundles/${LANE_ID}" -print0)
          if [[ "${#summaries[@]}" != "1" || "${#bundles[@]}" != "1" ]]; then
            echo "::${annotation}::Kova summary or bundle is missing for ${LANE_ID}."
            exit 1
          fi
          summary="$(realpath "${summaries[0]}")"
          bundle="$(realpath "${bundles[0]}")"
          if [[ "$summary" != "$input_root/"* || "$bundle" != "$input_root/"* || ! -s "$summary" ]]; then
            echo "::${annotation}::Kova summary or bundle escaped the downloaded artifact."
            exit 1
          fi
          if find "$bundle" -type l -print -quit | grep -q . || find "$bundle" ! -type d ! -type f -print -quit | grep -q .; then
            echo "::${annotation}::Kova bundle contains a symlink or special file."
            exit 1
          fi

          source_dir=""
          if [[ "$LANE_ID" == "mock-provider" ]]; then
            if [[ "$SOURCE_PRODUCER_ATTEMPT" == "0" ]]; then
              echo "::${annotation}::Source probe artifact attempt metadata is missing."
              exit 1
            fi
            # The mock lane downloads two IDs, so its source artifact is nested
            # under the immutable artifact name rather than extracted directly.
            source_path="${input_root}/openclaw-performance-source-${GITHUB_RUN_ID}-${SOURCE_PRODUCER_ATTEMPT}/${LANE_ID}"
            if [[ ! -d "$source_path" ]]; then
              echo "::${annotation}::Source probe artifact is missing for ${LANE_ID}."
              exit 1
            fi
            source_dir="$(realpath "$source_path")"
            if [[ "$source_dir" != "$input_root/"* ]] || find "$source_dir" -type l -print -quit | grep -q . || find "$source_dir" ! -type d ! -type f -print -quit | grep -q .; then
              echo "::${annotation}::Source probe artifact contains an unsafe path."
              exit 1
            fi
          elif [[ "$SOURCE_PRODUCER_ATTEMPT" != "0" ]]; then
            echo "::${annotation}::Unexpected source probe artifact attempt for ${LANE_ID}."
            exit 1
          fi

          git -C "$reports_root" init -b main
          git -C "$reports_root" config core.hooksPath /dev/null
          git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
          if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then
            echo "::${annotation}::Unable to fetch openclaw/clawgrit-reports before publishing."
            exit 1
          fi
          git -C "$reports_root" checkout -B main FETCH_HEAD

          ref_slug="$(printf '%s' "$TESTED_REF" | tr -c 'A-Za-z0-9._-' '-')"
          if [[ -z "$ref_slug" || "$ref_slug" == "." || "$ref_slug" == ".." ]]; then
            echo "::${annotation}::Tested ref does not produce a safe report path."
            exit 1
          fi
          run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}"
          dest_rel="openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
          report_url="https://github.com/openclaw/clawgrit-reports/tree/main/${dest_rel}"
          if git -C "$reports_root" cat-file -e "HEAD:${dest_rel}/report.json" 2>/dev/null; then
            {
              echo "### Clawgrit report already published"
              echo
              echo "- Published report: ${report_url}"
            } >> "$GITHUB_STEP_SUMMARY"
            echo "already_published=true" >> "$GITHUB_OUTPUT"
            echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
            echo "ready=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          dest="${reports_root}/${dest_rel}"
          mkdir -p "$dest"
          install -m 0644 "$report_json" "$dest/report.json"
          if [[ -n "$report_md" ]]; then
            install -m 0644 "$report_md" "$dest/report.md"
          fi
          install -m 0644 "$summary" "$dest/index.md"
          if [[ -n "$source_dir" ]]; then
            mkdir -p "$dest/source"
            cp -R "$source_dir"/. "$dest/source/"
            if [[ -f "$source_dir/index.md" ]]; then
              cat >> "$dest/index.md" <<'EOF'

          ## Source probes

          Additional gateway boot, memory, plugin pressure, mock hello-loop, CLI startup, and SQLite state smoke numbers are in [source/index.md](source/index.md).
          EOF
            fi
          fi

          artifact_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${ARTIFACT_ID}"
          {
            echo
            echo "## Full diagnostic artifact"
            echo
            echo "The complete Kova bundle remains in [Actions artifact ${ARTIFACT_ID}](${artifact_url}); its checksum is published under the bundles directory."
          } >> "$dest/index.md"

          jq -n \
            --arg repository "$GITHUB_REPOSITORY" \
            --arg ref "$TESTED_REF" \
            --arg sha "$TESTED_SHA" \
            --arg workflow_ref "$GITHUB_REF_NAME" \
            --arg workflow_sha "$GITHUB_SHA" \
            --arg workflow "$GITHUB_WORKFLOW" \
            --arg run_id "$GITHUB_RUN_ID" \
            --arg run_attempt "$PRODUCER_ATTEMPT" \
            --arg lane "$LANE_ID" \
            --arg path "$dest_rel" \
            '{repository:$repository,ref:$ref,sha:$sha,tested_ref:$ref,tested_sha:$sha,workflow_ref:$workflow_ref,workflow_sha:$workflow_sha,workflow:$workflow,run_id:$run_id,run_attempt:$run_attempt,lane:$lane,path:$path}' \
            > "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json"

          node "$PERFORMANCE_PUBLISHER_HELPER" \
            --bundle-dir "$bundle" \
            --bundle-destination "$dest/bundles" \
            --publish-root "$dest" \
            --max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES"

          git -C "$reports_root" config user.name "openclaw-performance[bot]"
          git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com"
          git -C "$reports_root" add -- openclaw-performance
          if ! git -C "$reports_root" diff --cached --quiet; then
            git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}"
          fi
          report_commit="$(git -C "$reports_root" rev-parse HEAD)"
          echo "already_published=false" >> "$GITHUB_OUTPUT"
          echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
          echo "report_commit=$report_commit" >> "$GITHUB_OUTPUT"
          echo "dest_rel=$dest_rel" >> "$GITHUB_OUTPUT"
          echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
          echo "ready=true" >> "$GITHUB_OUTPUT"

      - name: Create clawgrit reports app token
        id: clawgrit_app_token
        if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
        continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
        with:
          client-id: Iv23liOECG0slfuhz093
          private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}
          owner: openclaw
          repositories: clawgrit-reports
          permission-contents: write

      - name: Publish to clawgrit reports
        if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
        continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
        env:
          CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }}
          REPORTS_ROOT: ${{ steps.prepare.outputs.reports_root }}
          REPORT_COMMIT: ${{ steps.prepare.outputs.report_commit }}
          DEST_REL: ${{ steps.prepare.outputs.dest_rel }}
          REPORT_URL: ${{ steps.prepare.outputs.report_url }}
        shell: bash
        run: |
          set -euo pipefail
          annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
          if [[ -z "${CLAWGRIT_REPORTS_APP_TOKEN:-}" ]]; then
            {
              echo "### Clawgrit report publish unavailable"
              echo
              echo "The repository-scoped ClawSweeper GitHub App token could not be created."
            } >> "$GITHUB_STEP_SUMMARY"
            echo "::${annotation}::ClawSweeper GitHub App token is unavailable; Kova artifacts were uploaded, but the report cannot be published."
            exit 1
          fi
          token="$CLAWGRIT_REPORTS_APP_TOKEN"
          unset CLAWGRIT_REPORTS_APP_TOKEN
          auth_header="$(printf 'x-access-token:%s' "$token" | base64 | tr -d '\n')"
          unset token
          echo "::add-mask::${auth_header}"

          reports_root="$(realpath "$REPORTS_ROOT")"
          if [[ "$reports_root" != "$RUNNER_TEMP/"* || ! "$REPORT_COMMIT" =~ ^[0-9a-f]{40}$ || ! "$DEST_REL" =~ ^openclaw-performance/[A-Za-z0-9._-]+/[0-9]+-[0-9]+/[A-Za-z0-9._-]+$ ]]; then
            echo "::${annotation}::Prepared report metadata failed validation."
            exit 1
          fi
          if [[ "$REPORT_URL" != "https://github.com/openclaw/clawgrit-reports/tree/main/${DEST_REL}" ]]; then
            echo "::${annotation}::Prepared report URL failed validation."
            exit 1
          fi

          git_local() {
            git -C "$reports_root" -c core.hooksPath=/dev/null "$@"
          }
          push_report() {
            GIT_TERMINAL_PROMPT=0               GIT_CONFIG_COUNT=2               GIT_CONFIG_KEY_0=core.hooksPath               GIT_CONFIG_VALUE_0=/dev/null               GIT_CONFIG_KEY_1=http.https://github.com/.extraheader               GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}"               timeout 120s git -C "$reports_root" push origin HEAD:main
          }
          write_publish_success() {
            {
              echo "### Clawgrit report published"
              echo
              echo "- Published report: ${REPORT_URL}"
            } >> "$GITHUB_STEP_SUMMARY"
          }
          write_publish_failure() {
            {
              echo "### Clawgrit report publish failed"
              echo
              echo "Kova artifacts were uploaded, but publishing the clawgrit report failed after $1 attempts."
              echo "Check the ClawSweeper GitHub App installation and contents permission for openclaw/clawgrit-reports."
            } >> "$GITHUB_STEP_SUMMARY"
            echo "::${annotation}::Kova artifacts uploaded, but clawgrit report publish failed after $1 attempts."
          }

          if [[ "$(git_local config --local --get core.hooksPath)" != "/dev/null" || "$(git_local remote get-url origin)" != "https://github.com/openclaw/clawgrit-reports.git" ]]; then
            echo "::${annotation}::Prepared reports checkout failed integrity validation."
            exit 1
          fi
          git_local cat-file -e "${REPORT_COMMIT}^{commit}"
          report_commit="$REPORT_COMMIT"

          for attempt in 1 2 3 4 5; do
            if push_report; then
              write_publish_success
              exit 0
            fi

            sleep $((attempt * 2))
            if ! timeout 120s git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main; then
              if [[ "$attempt" != "5" ]]; then
                echo "::warning::Unable to refresh openclaw/clawgrit-reports after publish attempt ${attempt}; retrying."
                continue
              fi
            else
              if git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json" 2>/dev/null; then
                # A timed-out push may have succeeded; verify its unique report before failing.
                write_publish_success
                exit 0
              fi
              if [[ "$attempt" != "5" ]]; then
                git_local checkout --detach FETCH_HEAD
                # The run path is unique; only the shared latest pointer can conflict.
                # Replaying with the current commit wins that pointer deterministically.
                if ! git_local cherry-pick -X theirs "$report_commit"; then
                  git_local cherry-pick --abort || true
                  echo "::${annotation}::Unable to replay the clawgrit report after a concurrent publish."
                  exit 1
                fi
                report_commit="$(git_local rev-parse HEAD)"
                continue
              fi
            fi

            write_publish_failure "$attempt"
            exit 1
          done

  artifact_only_guard:
    name: Verify artifact-only report mode
    needs: [resolve_target, kova, publish]
    if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.publish_reports != true }}
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    permissions:
      contents: read
    steps:
      - name: Verify report publisher stayed disabled
        env:
          PUBLISH_RESULT: ${{ needs.publish.result }}
        shell: bash
        run: |
          set -euo pipefail
          if [[ "$PUBLISH_RESULT" != "skipped" ]]; then
            echo "::error::Artifact-only performance mode requires the report publisher to stay skipped; got ${PUBLISH_RESULT}."
            exit 1
          fi
          {
            echo "### Artifact-only report mode"
            echo
            echo "Clawgrit report publication stayed disabled; Kova evidence is available only as workflow artifacts."
          } >> "$GITHUB_STEP_SUMMARY"