Skip to content
OpenClaw Performance

OpenClaw Performance #95

Sign in to view logs

OpenClaw Performance

OpenClaw Performance #95

Workflow file for this run

.github/workflows/openclaw-performance.yml at 26a58bc
name: OpenClaw Performance
run-name: ${{ inputs.dispatch_id != '' && format('OpenClaw Performance {0}', inputs.dispatch_id) || 'OpenClaw Performance' }}
on:
schedule:
- cron: "11 5 * * *"
workflow_dispatch:
inputs:
target_ref:
description: OpenClaw ref to benchmark; defaults to the workflow ref
required: false
default: ""
type: string
profile:
description: Kova profile to run
required: false
default: diagnostic
type: choice
options:
- smoke
- diagnostic
- soak
- release
repeat:
description: Repeat count for non-profiled Kova runs
required: false
default: "3"
type: string
deep_profile:
description: Run the deep-profile lane with CPU/heap/trace artifacts
required: false
default: false
type: boolean
live_openai_candidate:
description: Run the live OpenAI GPT 5.6 agent-turn lane
required: false
default: false
type: boolean
fail_on_regression:
description: Fail the workflow when Kova exits non-zero
required: false
default: false
type: boolean
publish_reports:
description: Publish completed reports to openclaw/clawgrit-reports
required: false
default: true
type: boolean
kova_ref:
description: Optional openclaw/Kova Git ref override; defaults to the fixture compatible with the target config schema
required: false
default: ""
type: string
kova_config_contract:
description: Optional fixture-contract override for a custom Kova ref
required: false
default: ""
type: string
dispatch_id:
description: Optional parent workflow dispatch identifier
required: false
default: ""
type: string
permissions:
contents: read
concurrency:
group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-{1}', github.workflow, github.run_id) || format('{0}-{1}', github.workflow, github.ref) }}
cancel-in-progress: false
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
OCM_VERSION: v0.2.29
OCM_LINUX_X64_SHA256: d966098d6ba2bc10891be3c76e162a37b07f28c4f51da75d2eb509886eb7e1cf
KOVA_REPOSITORY: openclaw/Kova
KOVA_CANONICAL_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78
KOVA_LEGACY_LIST_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78
PERFORMANCE_MODEL_ID: gpt-5.6-luna
# Release matrices cold-build the candidate runtime before measurement.
KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }}
jobs:
resolve_target:
name: Resolve target ref
runs-on: ubuntu-24.04
timeout-minutes: 10
outputs:
checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
tested_ref: ${{ steps.resolve.outputs.tested_ref }}
tested_sha: ${{ steps.resolve.outputs.tested_sha }}
kova_ref: ${{ steps.resolve.outputs.kova_ref }}
kova_config_contract: ${{ steps.resolve.outputs.kova_config_contract }}
kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }}
steps:
- name: Resolve OpenClaw target ref
id: resolve
env:
GH_TOKEN: ${{ github.token }}
TARGET_REF_INPUT: ${{ inputs.target_ref }}
KOVA_REF_INPUT: ${{ inputs.kova_ref }}
KOVA_CONFIG_CONTRACT_INPUT: ${{ inputs.kova_config_contract }}
shell: bash
run: |
set -euo pipefail
requested="${TARGET_REF_INPUT:-}"
if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then
echo "::error::target_ref must be a single line."
exit 1
fi
if [[ -z "$requested" ]]; then
resolved_sha="$GITHUB_SHA"
tested_ref="$GITHUB_REF_NAME"
else
encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")"
if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then
echo "::error::Unable to resolve OpenClaw target_ref '${requested}'."
exit 1
fi
tested_ref="$requested"
fi
if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'."
exit 1
fi
kova_ref="${KOVA_REF_INPUT:-}"
kova_config_contract="${KOVA_CONFIG_CONTRACT_INPUT:-}"
if [[ "$kova_ref" == *$'\n'* || "$kova_ref" == *$'\r'* ]]; then
echo "::error::kova_ref must be a single line."
exit 1
fi
if [[ "$kova_config_contract" == *$'\n'* || "$kova_config_contract" == *$'\r'* ]]; then
echo "::error::kova_config_contract must be a single line."
exit 1
fi
if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then
if schema_content="$({
gh api "repos/${GITHUB_REPOSITORY}/contents/src/config/zod-schema.agent-defaults.ts?ref=${resolved_sha}" --jq '.content' |
base64 --decode
})"; then
:
elif [[ -z "$kova_ref" ]]; then
echo "::error::Unable to inspect the Kova config-fixture contract for target ${resolved_sha}. Supply kova_ref explicitly and optionally set kova_config_contract for that producer."
exit 1
else
schema_content=""
fi
if [[ -n "$schema_content" ]] && grep -Fqx ' mediaModels: z' <<<"$schema_content"; then
detected_kova_config_contract="canonical"
default_kova_ref="$KOVA_CANONICAL_CONFIG_REF"
elif [[ -n "$schema_content" ]] && grep -Fqx ' imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then
detected_kova_config_contract="legacy-list"
default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF"
else
detected_kova_config_contract=""
default_kova_ref=""
fi
if [[ -n "$detected_kova_config_contract" ]]; then
kova_config_contract="${kova_config_contract:-$detected_kova_config_contract}"
kova_ref="${kova_ref:-$default_kova_ref}"
elif [[ -z "$kova_ref" ]]; then
echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly and optionally set kova_config_contract for that producer."
exit 1
fi
fi
echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT"
echo "tested_ref=$tested_ref" >> "$GITHUB_OUTPUT"
echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT"
echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT"
echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"
if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then
echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"
else
echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT"
fi
kova:
name: ${{ matrix.title }}
needs: resolve_target
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
include:
- lane: mock-provider
title: Kova mock provider performance
auth: mock
repeat: input
deep_profile: "false"
live: "false"
managed_service: "true"
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:agent-cold-warm-message"
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,bundled-plugin-startup:fresh,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
- lane: mock-deep-profile
title: Kova mock provider deep profile
auth: mock
repeat: "1"
deep_profile: "true"
live: "false"
managed_service: "true"
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message"
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
- lane: live-openai-candidate
title: Kova live OpenAI GPT 5.6 agent turn
auth: live
repeat: "1"
deep_profile: "false"
live: "true"
managed_service: "false"
include_filters: "scenario:agent-cold-warm-message"
expected_release_entries: "agent-cold-warm-message:mock-openai-provider"
env:
KOVA_REF: ${{ needs.resolve_target.outputs.kova_ref }}
KOVA_OPENCLAW_CONFIG_CONTRACT: ${{ needs.resolve_target.outputs.kova_config_contract }}
KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }}
PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }}
BUNDLE_DIR: ${{ github.workspace }}/.artifacts/kova/bundles/${{ matrix.lane }}
SUMMARY_DIR: ${{ github.workspace }}/.artifacts/kova/summaries
LANE_ID: ${{ matrix.lane }}
TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
PROFILE: ${{ inputs.profile || 'diagnostic' }}
REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
FAIL_ON_REGRESSION: ${{ inputs.fail_on_regression || 'false' }}
INCLUDE_FILTERS: ${{ matrix.include_filters }}
EXPECTED_RELEASE_ENTRIES: ${{ matrix.expected_release_entries }}
AUTH_MODE: ${{ matrix.auth }}
MATRIX_REPEAT: ${{ matrix.repeat }}
MATRIX_DEEP_PROFILE: ${{ matrix.deep_profile }}
MATRIX_LIVE: ${{ matrix.live }}
KOVA_REF_TRUSTED_FOR_LIVE: ${{ needs.resolve_target.outputs.kova_ref_trusted_for_live }}
steps:
- name: Decide lane
id: lane
shell: bash
run: |
set -euo pipefail
run_lane=true
reason=""
if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
run_lane=false
reason="deep_profile input is false"
fi
if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
run_lane=false
reason="live_openai_candidate input is false"
fi
if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then
echo "::error::The live OpenAI lane only executes a reviewed immutable Kova default. Omit kova_ref or update the pinned workflow defaults after review."
exit 1
fi
echo "run=$run_lane" >> "$GITHUB_OUTPUT"
if [[ "$run_lane" != "true" ]]; then
echo "Skipping ${LANE_ID}: ${reason}" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Checkout OpenClaw
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ needs.resolve_target.outputs.checkout_ref }}
fetch-depth: 1
persist-credentials: false
- name: Checkout performance workflow helpers
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.sha }}
path: .artifacts/performance-workflow
fetch-depth: 1
persist-credentials: false
- name: Record tested revision
if: steps.lane.outputs.run == 'true'
shell: bash
run: |
set -euo pipefail
tested_sha="$(git rev-parse HEAD)"
if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
exit 1
fi
echo "TESTED_REF=${TARGET_REF}" >> "$GITHUB_ENV"
echo "TESTED_SHA=${EXPECTED_TESTED_SHA}" >> "$GITHUB_ENV"
{
echo "Tested ref: ${TARGET_REF}"
echo "Tested SHA: ${EXPECTED_TESTED_SHA}"
echo "Workflow ref: ${GITHUB_REF_NAME}"
echo "Workflow SHA: ${GITHUB_SHA}"
} >> "$GITHUB_STEP_SUMMARY"
- name: Set up Node environment
if: steps.lane.outputs.run == 'true'
uses: ./.github/actions/setup-node-env
with:
install-bun: "false"
- name: Prepare systemd user session
if: ${{ steps.lane.outputs.run == 'true' && matrix.managed_service == 'true' }}
shell: bash
run: |
set -euo pipefail
user="$(id -un)"
uid="$(id -u)"
test "$(ps -p 1 -o comm= | xargs)" = systemd
sudo systemctl is-active --quiet systemd-logind.service
sudo loginctl enable-linger "$user"
sudo systemctl start "user@${uid}.service"
runtime_dir="$(loginctl show-user "$user" --property=RuntimePath --value)"
test -n "$runtime_dir"
test -d "$runtime_dir"
test -O "$runtime_dir"
export XDG_RUNTIME_DIR="$runtime_dir"
test -S "$XDG_RUNTIME_DIR/systemd/private"
echo "XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR" >> "$GITHUB_ENV"
if [[ -S "$runtime_dir/bus" ]]; then
export DBUS_SESSION_BUS_ADDRESS="unix:path=${runtime_dir}/bus"
echo "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" >> "$GITHUB_ENV"
fi
systemctl --user show-environment >/dev/null
- name: Install OCM and Kova
if: steps.lane.outputs.run == 'true'
shell: bash
run: |
set -euo pipefail
KOVA_SRC="${RUNNER_TEMP}/kova-src"
echo "KOVA_SRC=$KOVA_SRC" >> "$GITHUB_ENV"
mkdir -p "$HOME/.local/bin" "$(dirname "$KOVA_SRC")" "${RUNNER_TEMP}/ocm-install"
ocm_archive="${RUNNER_TEMP}/ocm-${OCM_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
# retry-max-time only gates new retries; cap an active transfer separately
# so a stalled release asset cannot consume the entire Kova job deadline.
curl -fsSL --proto '=https' --tlsv1.2 \
--max-time 180 \
--retry 8 --retry-max-time 180 --retry-all-errors --retry-connrefused \
-o "$ocm_archive" \
"https://github.com/shakkernerd/ocm/releases/download/${OCM_VERSION}/ocm-x86_64-unknown-linux-gnu.tar.gz"
echo "${OCM_LINUX_X64_SHA256} ${ocm_archive}" | sha256sum -c -
tar -xzf "$ocm_archive" -C "${RUNNER_TEMP}/ocm-install"
install -m 0755 "${RUNNER_TEMP}/ocm-install/ocm" "$HOME/.local/bin/ocm"
git init -b main "$KOVA_SRC"
git -C "$KOVA_SRC" remote add origin "https://github.com/${KOVA_REPOSITORY}.git"
git -C "$KOVA_SRC" fetch --filter=blob:none --depth 1 origin "$KOVA_REF"
git -C "$KOVA_SRC" checkout --detach FETCH_HEAD
npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund
node - "$KOVA_SRC" <<'NODE'
const fs = require("node:fs");
const path = require("node:path");
const root = process.argv[2];
const packageJsonPath = require.resolve("mock-ai-provider/package.json", {
paths: [root],
});
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
const bin =
typeof packageJson.bin === "string"
? packageJson.bin
: packageJson.bin?.["mock-ai-provider"];
if (typeof bin !== "string" || bin.length === 0) {
throw new Error("mock-ai-provider does not declare its CLI entrypoint");
}
fs.accessSync(path.resolve(path.dirname(packageJsonPath), bin), fs.constants.R_OK);
fs.accessSync(
path.join(root, "node_modules", ".bin", "mock-ai-provider"),
fs.constants.X_OK,
);
require.resolve("zod", { paths: [root] });
NODE
cat > "$HOME/.local/bin/kova" <<EOF
#!/usr/bin/env bash
export KOVA_HOME="${KOVA_HOME}"
exec node "${KOVA_SRC}/bin/kova.mjs" "\$@"
EOF
chmod 0755 "$HOME/.local/bin/kova"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Configure OCM local workspace dependencies
if: steps.lane.outputs.run == 'true'
shell: bash
run: |
set -euo pipefail
npm_wrapper="$PERFORMANCE_HELPER_DIR/scripts/ocm-npm-workspace-deps.mjs"
workspace_dependency_dirs=""
if [[ -f "${GITHUB_WORKSPACE}/packages/ai/package.json" ]]; then
workspace_dependency_dirs="${GITHUB_WORKSPACE}/packages/ai"
fi
chmod 0755 "$npm_wrapper"
{
echo "OCM_INTERNAL_NPM_BIN=$npm_wrapper"
echo "OPENCLAW_OCM_REAL_NPM_BIN=$(command -v npm)"
echo "OPENCLAW_OCM_WORKSPACE_DEPENDENCY_DIRS=$workspace_dependency_dirs"
} >> "$GITHUB_ENV"
- name: Kova version and plan sanity
if: steps.lane.outputs.run == 'true'
shell: bash
run: |
set -euo pipefail
repeat="$REQUESTED_REPEAT"
if [[ "$MATRIX_REPEAT" != "input" ]]; then
repeat="$MATRIX_REPEAT"
fi
plan_dir="${RUNNER_TEMP}/kova-plans"
plan_json="${plan_dir}/${LANE_ID}.json"
mkdir -p "$plan_dir"
kova version --json
kova matrix plan \
--profile "$PROFILE" \
--target "local-build:${GITHUB_WORKSPACE}" \
--include "$INCLUDE_FILTERS" \
--parallel 1 \
--repeat "$repeat" \
--json >"$plan_json"
node --input-type=module - "$plan_json" <<'NODE'
import fs from "node:fs";
const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const expectedFilters = process.env.INCLUDE_FILTERS.split(",");
const actualFilters = plan.controls?.include;
if (
!Array.isArray(actualFilters) ||
actualFilters.length !== expectedFilters.length ||
actualFilters.some((filter, index) => filter !== expectedFilters[index])
) {
throw new Error("Kova plan did not preserve the requested include filters");
}
if (process.env.PROFILE === "release") {
const expectedEntries = process.env.EXPECTED_RELEASE_ENTRIES.split(",").sort();
if (!Array.isArray(plan.entries)) {
throw new Error("Kova release plan did not contain entries");
}
const actualEntries = plan.entries
.map((entry) => {
if (entry.status !== "SELECTED" || !entry.scenario?.id || !entry.state?.id) {
throw new Error("Kova release plan contained an invalid selected entry");
}
return `${entry.scenario.id}:${entry.state.id}`;
})
.sort();
if (
actualEntries.length !== expectedEntries.length ||
actualEntries.some((entry, index) => entry !== expectedEntries[index])
) {
throw new Error("Kova release plan entries did not match the required lane coverage");
}
}
NODE
echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"
echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV"
- name: Configure live OpenAI auth
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
shell: bash
run: |
set -euo pipefail
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
echo "OPENAI_API_KEY is not configured; live GPT 5.6 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY"
exit 1
fi
kova setup --ci --json
kova setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY --json
- name: Run Kova
id: kova
if: steps.lane.outputs.run == 'true'
env:
OPENAI_API_KEY: ${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}
shell: bash
run: |
set -euo pipefail
mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR"
repeat="$KOVA_LANE_REPEAT"
args=(
matrix run
--profile "$PROFILE"
--target "local-build:${GITHUB_WORKSPACE}"
--include "$INCLUDE_FILTERS"
--auth "$AUTH_MODE"
--parallel 1
--repeat "$repeat"
--timeout-ms "$KOVA_SCENARIO_TIMEOUT_MS"
--report-dir "$REPORT_DIR"
--execute
--json
)
if [[ "$AUTH_MODE" == "live" ]]; then
args+=(--model "$PERFORMANCE_MODEL_ID")
fi
if [[ "$MATRIX_DEEP_PROFILE" == "true" ]]; then
args+=(--deep-profile)
fi
if [[ "$FAIL_ON_REGRESSION" == "true" ]]; then
args+=(--gate)
fi
log_path="$REPORT_DIR/${LANE_ID}.log"
set +e
kova "${args[@]}" 2>&1 | tee "$log_path"
status=${PIPESTATUS[0]}
set -e
report_json="$(node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR")"
report_md="${report_json%.json}.md"
echo "status=$status" >> "$GITHUB_OUTPUT"
echo "report_json=$report_json" >> "$GITHUB_OUTPUT"
echo "report_md=$report_md" >> "$GITHUB_OUTPUT"
set +e
node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \
--plan "$KOVA_PLAN_JSON" \
--report "$report_json" \
--profile "$PROFILE" \
--target "local-build:${GITHUB_WORKSPACE}" \
--repeat "$repeat" \
--include "$INCLUDE_FILTERS" \
--auth "$AUTH_MODE" \
--model "$PERFORMANCE_MODEL_ID"
evidence_status=$?
set -e
effective_status="$status"
if [[ "$evidence_status" == "0" && "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
gate_args=("$report_json")
if [[ "$KOVA_REF" == "$KOVA_CANONICAL_CONFIG_REF" || "$KOVA_REF" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then
gate_args+=(--require-instrumented-performance-contract)
fi
if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "${gate_args[@]}"
then
effective_status=0
{
echo "Kova returned a non-zero release-gate verdict, but the trusted report adapter found only filtered coverage or profiling-affected resource thresholds with no baseline regression."
echo
} >> "$GITHUB_STEP_SUMMARY"
fi
fi
echo "effective_status=$effective_status" >> "$GITHUB_OUTPUT"
set +e
kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json"
bundle_status=${PIPESTATUS[0]}
set -e
summary_path="$SUMMARY_DIR/${LANE_ID}.md"
summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mjs" --report "$report_json" --output "$summary_path" --lane "$LANE_ID")
set +e
"${summary_args[@]}"
summary_status=$?
set -e
if [[ "$summary_status" != "0" ]]; then
cat > "$summary_path" <<EOF
# OpenClaw Performance Report
- Lane: ${LANE_ID}
- Kova report: ${report_json}
- Summary generation failed with status ${summary_status}; inspect the Kova report and lane log.
EOF
fi
cat >> "$summary_path" <<EOF
## Test scope
- Repository: ${GITHUB_REPOSITORY}
- Tested ref: ${TESTED_REF}
- Tested SHA: ${TESTED_SHA}
- Workflow ref: ${GITHUB_REF_NAME}
- Workflow SHA: ${GITHUB_SHA}
- Kova repository: ${KOVA_REPOSITORY}
- Kova ref: ${KOVA_REF}
- Kova profile: ${PROFILE}
- Kova scenario timeout: ${KOVA_SCENARIO_TIMEOUT_MS}ms
- Lane auth: ${AUTH_MODE}
- Lane model: ${PERFORMANCE_MODEL_ID}
- Lane repeat: ${repeat}
- Include filters: ${INCLUDE_FILTERS}
EOF
cat "$summary_path" >> "$GITHUB_STEP_SUMMARY"
if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]; then
exit 1
fi
if [[ "$FAIL_ON_REGRESSION" == "true" && "$effective_status" != "0" ]]; then
exit "$effective_status"
fi
- name: Validate Kova evidence
if: ${{ always() && steps.lane.outputs.run == 'true' }}
shell: bash
run: |
set -euo pipefail
missing=0
if ! node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR" >/dev/null; then
echo "::error::Exactly one full Kova JSON report is required for ${LANE_ID}."
missing=1
fi
if [[ ! -s "$BUNDLE_DIR/bundle.json" ]]; then
echo "::error::Kova bundle evidence is missing for ${LANE_ID}."
missing=1
fi
if [[ ! -s "$SUMMARY_DIR/${LANE_ID}.md" ]]; then
echo "::error::Kova summary evidence is missing for ${LANE_ID}."
missing=1
fi
if [[ "$missing" != "0" ]]; then
exit 1
fi
- name: Upload Kova artifacts
if: ${{ always() && steps.lane.outputs.run == 'true' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: openclaw-performance-${{ matrix.lane }}-${{ github.run_id }}-${{ github.run_attempt }}
path: |
.artifacts/kova/reports/${{ matrix.lane }}
.artifacts/kova/bundles/${{ matrix.lane }}
.artifacts/kova/summaries/${{ matrix.lane }}.md
if-no-files-found: error
retention-days: ${{ matrix.deep_profile == 'true' && 14 || 30 }}
source_performance:
name: OpenClaw source performance probes
needs: resolve_target
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 120
env:
PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
SOURCE_PERF_DIR: ${{ github.workspace }}/.artifacts/openclaw-performance/source/mock-provider
TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
steps:
- name: Checkout OpenClaw source target
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ needs.resolve_target.outputs.checkout_ref }}
fetch-depth: 1
persist-credentials: false
- name: Checkout source performance helpers
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.sha }}
path: .artifacts/performance-workflow
fetch-depth: 1
persist-credentials: false
- name: Record source performance revision
shell: bash
run: |
set -euo pipefail
tested_sha="$(git rev-parse HEAD)"
if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
exit 1
fi
{
echo "Tested ref: ${TESTED_REF}"
echo "Tested SHA: ${TESTED_SHA}"
echo "Workflow ref: ${GITHUB_REF_NAME}"
echo "Workflow SHA: ${GITHUB_SHA}"
} >> "$GITHUB_STEP_SUMMARY"
- name: Set up source performance environment
uses: ./.github/actions/setup-node-env
with:
install-bun: "false"
- name: Fetch previous source performance baseline
shell: bash
run: |
set -euo pipefail
reports_root=".artifacts/clawgrit-baseline"
mkdir -p "$reports_root"
git -C "$reports_root" init -b main
git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
if ! git -C "$reports_root" fetch --filter=blob:none --depth=1 origin main; then
echo "No previous source performance baseline could be fetched." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')"
pointer="openclaw-performance/${ref_slug}/latest-mock-provider.json"
if ! git -C "$reports_root" cat-file -e "FETCH_HEAD:${pointer}"; then
echo "No previous source performance baseline exists for ${TESTED_REF}." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
if ! latest_path="$(git -C "$reports_root" show "FETCH_HEAD:${pointer}" | node -e "const fs=require('node:fs'); const data=JSON.parse(fs.readFileSync(0,'utf8')); const value=String(data.path || ''); if (!/^openclaw-performance\\/[A-Za-z0-9._-]+\\/[0-9]+-[0-9]+\\/mock-provider$/u.test(value)) process.exit(1); process.stdout.write(value);")"; then
echo "Previous source performance baseline pointer is invalid." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
git -C "$reports_root" sparse-checkout init --no-cone
printf '/%s/source/\n' "$latest_path" | git -C "$reports_root" sparse-checkout set --stdin
git -C "$reports_root" checkout --detach FETCH_HEAD
baseline_source="${reports_root}/${latest_path}/source"
if [[ -d "$baseline_source" ]]; then
baseline_source="$(realpath "$baseline_source")"
echo "SOURCE_PERF_BASELINE_DIR=$baseline_source" >> "$GITHUB_ENV"
echo "Using source performance baseline: ${latest_path}/source" >> "$GITHUB_STEP_SUMMARY"
else
echo "Previous source performance baseline has no source directory." >> "$GITHUB_STEP_SUMMARY"
fi
- name: Run OpenClaw source performance probes
shell: bash
run: |
set -euo pipefail
source_runs="$REQUESTED_REPEAT"
if ! [[ "$source_runs" =~ ^[0-9]+$ ]] || [[ "$source_runs" -lt 1 ]]; then
source_runs=3
fi
mkdir -p "$SOURCE_PERF_DIR/mock-hello"
if ! node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:gateway:cpu-scenarios'] && scripts['test:extensions:memory'] && scripts.openclaw && fs.existsSync('openclaw.mjs') && fs.existsSync('scripts/profile-extension-memory.mjs') ? 0 : 1)"; then
cat > "$SOURCE_PERF_DIR/index.md" <<EOF
# OpenClaw Source Performance
Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)
Source probes skipped for this tested ref because one or more probe entry points are not present in the checked-out source tree.
## Test scope
- Tested ref: ${TESTED_REF}
- Tested SHA: ${TESTED_SHA}
- Required scripts: test:gateway:cpu-scenarios, test:extensions:memory, openclaw, openclaw.mjs, scripts/profile-extension-memory.mjs
EOF
cat "$SOURCE_PERF_DIR/index.md" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
# target_ref may predate the dedicated profile; preserve the historical full build there.
if node -e "import('./scripts/build-all.mjs').then((module) => process.exit(module.BUILD_ALL_PROFILES?.sourcePerformance ? 0 : 1)).catch(() => process.exit(1))"; then
OPENCLAW_BUILD_PRIVATE_QA=1 node scripts/build-all.mjs sourcePerformance
else
pnpm build
fi
supported_startup_cases="$(
node --import tsx scripts/bench-gateway-startup.ts --help |
sed -n 's/^ \([[:alnum:]_-][[:alnum:]_-]*\) (.*/\1/p'
)"
startup_case_args=()
for startup_case in default skipChannels preparedRuntimeCatalogStall preparedRuntimeScaleOne preparedRuntimeScaleMany oneInternalHook allInternalHooks fiftyPlugins fiftyStartupLazyPlugins; do
if grep -Fxq "$startup_case" <<<"$supported_startup_cases"; then
startup_case_args+=(--startup-case "$startup_case")
fi
done
if [[ " ${startup_case_args[*]} " != *" --startup-case default "* ]]; then
echo "The target startup benchmark did not advertise its required default case." >&2
exit 1
fi
pnpm test:gateway:cpu-scenarios \
--output-dir "$SOURCE_PERF_DIR/gateway-cpu" \
--runs "$source_runs" \
--warmup 1 \
--skip-qa \
"${startup_case_args[@]}"
pnpm test:extensions:memory \
-- --json "$SOURCE_PERF_DIR/extension-memory.json"
for run_index in $(seq 1 "$source_runs"); do
run_dir="$SOURCE_PERF_DIR/mock-hello/run-$(printf '%03d' "$run_index")"
pnpm openclaw qa suite \
--provider-mode mock-openai \
--model "mock-openai/${PERFORMANCE_MODEL_ID}" \
--concurrency 1 \
--output-dir "$(realpath --relative-to="$GITHUB_WORKSPACE" "$run_dir")" \
--scenario channel-chat-baseline
done
gateway_home="$(mktemp -d)"
gateway_readiness_home="$(mktemp -d)"
gateway_port="$(node -e "const net=require('node:net'); const s=net.createServer(); s.listen(0,'127.0.0.1',()=>{ console.log(s.address().port); s.close(); });")"
gateway_state="$gateway_home/.openclaw"
gateway_config="$gateway_state/openclaw.json"
gateway_readiness_state="$gateway_readiness_home/.openclaw"
gateway_readiness_config="$gateway_readiness_state/openclaw.json"
gateway_log="$SOURCE_PERF_DIR/cli-gateway.log"
gateway_readiness_log="$SOURCE_PERF_DIR/cli-gateway-readiness.log"
gateway_pid=""
mkdir -p "$gateway_state" "$gateway_readiness_state"
catalog_refresh_config=""
# Frozen release candidates can predate this config key. Inspect the
# target schema so the trusted harness never writes invalid config.
if rg -q 'catalogRefresh:' src/config/zod-schema.core.ts; then
catalog_refresh_config=' "models": { "catalogRefresh": { "enabled": false } },'
fi
cat > "$gateway_config" <<EOF
{
"browser": { "enabled": false },
${catalog_refresh_config}
"update": { "checkOnStart": false },
"gateway": {
"mode": "local",
"port": ${gateway_port},
"bind": "loopback",
"auth": { "mode": "none" },
"controlUi": { "enabled": false },
"tailscale": { "mode": "off" }
},
"plugins": {
"enabled": true,
"entries": { "browser": { "enabled": false } }
}
}
EOF
cp "$gateway_config" "$gateway_readiness_config"
: > "$gateway_readiness_log"
cleanup_gateway() {
if [[ -n "${gateway_pid:-}" ]] && kill -0 "$gateway_pid" 2>/dev/null; then
kill "$gateway_pid" 2>/dev/null || true
wait "$gateway_pid" 2>/dev/null || true
fi
rm -rf "$gateway_home" "$gateway_readiness_home"
}
trap cleanup_gateway EXIT
OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" OPENCLAW_SKIP_CHANNELS=1 \
node dist/entry.js gateway run --bind loopback --port "$gateway_port" --auth none --allow-unconfigured --force \
>"$gateway_log" 2>&1 &
gateway_pid="$!"
gateway_ready_timeout_seconds=120
gateway_probe_timeout_seconds=5
gateway_ready_deadline=$((SECONDS + gateway_ready_timeout_seconds))
while true; do
gateway_ready_remaining=$((gateway_ready_deadline - SECONDS))
if (( gateway_ready_remaining <= 0 )); then
cat "$gateway_log" >&2
echo "Timed out after ${gateway_ready_timeout_seconds}s waiting for gateway health." >&2
exit 1
fi
# Clamp each probe to the remaining startup budget so a stalled response cannot multiply the wait.
gateway_probe_timeout="$gateway_ready_remaining"
if (( gateway_probe_timeout > gateway_probe_timeout_seconds )); then
gateway_probe_timeout="$gateway_probe_timeout_seconds"
fi
if curl -fsS --connect-timeout 2 --max-time "$gateway_probe_timeout" "http://127.0.0.1:${gateway_port}/healthz" >/dev/null; then
break
fi
if ! kill -0 "$gateway_pid" 2>/dev/null; then
cat "$gateway_log" >&2
exit 1
fi
sleep 1
done
# Exercise the real WebSocket handshake and health RPC without warming benchmark device state.
while true; do
gateway_ready_remaining=$((gateway_ready_deadline - SECONDS))
if (( gateway_ready_remaining <= 0 )); then
cat "$gateway_log" >&2
cat "$gateway_readiness_log" >&2
echo "Timed out after ${gateway_ready_timeout_seconds}s waiting for gateway WebSocket health." >&2
exit 1
fi
gateway_ready_remaining_ms=$((gateway_ready_remaining * 1000))
if OPENCLAW_HOME="$gateway_readiness_home" OPENCLAW_STATE_DIR="$gateway_readiness_state" OPENCLAW_CONFIG_PATH="$gateway_readiness_config" \
node dist/entry.js gateway health \
--port "$gateway_port" \
--timeout "$gateway_ready_remaining_ms" \
--json >"$gateway_readiness_log" 2>&1; then
break
fi
if ! kill -0 "$gateway_pid" 2>/dev/null; then
cat "$gateway_log" >&2
cat "$gateway_readiness_log" >&2
exit 1
fi
# Pace fast transport failures without extending the shared startup deadline.
gateway_ready_remaining=$((gateway_ready_deadline - SECONDS))
if (( gateway_ready_remaining > 0 )); then
sleep 1
fi
done
OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" \
node --import tsx "$PERFORMANCE_HELPER_DIR/scripts/bench-cli-startup.ts" \
--entry "$GITHUB_WORKSPACE/openclaw.mjs" \
--case gatewayHealthJsonConnected \
--case gatewayHealthJsonFirstDevice \
--case configGetGatewayPort \
--runs "$source_runs" \
--warmup 1 \
--output "$SOURCE_PERF_DIR/cli-startup.json"
cleanup_gateway
trap - EXIT
if node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:sqlite:perf:smoke'] && fs.existsSync('scripts/bench-sqlite-state.ts') ? 0 : 1)"; then
pnpm test:sqlite:perf:smoke
cp .artifacts/sqlite-perf/smoke.json "$SOURCE_PERF_DIR/sqlite-perf-smoke.json"
else
echo "SQLite state smoke probe is not available in ${TESTED_REF}; continuing with the remaining source probes." >> "$GITHUB_STEP_SUMMARY"
fi
summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/openclaw-performance-source-summary.mjs" \
--source-dir "$SOURCE_PERF_DIR" \
--output "$SOURCE_PERF_DIR/index.md")
if [[ -n "${SOURCE_PERF_BASELINE_DIR:-}" && -d "$SOURCE_PERF_BASELINE_DIR" ]]; then
summary_args+=(--baseline-source-dir "$SOURCE_PERF_BASELINE_DIR")
fi
"${summary_args[@]}"
cat "$SOURCE_PERF_DIR/index.md" >> "$GITHUB_STEP_SUMMARY"
- name: Upload source performance artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: openclaw-performance-source-${{ github.run_id }}-${{ github.run_attempt }}
path: .artifacts/openclaw-performance/source
if-no-files-found: error
retention-days: 30
publish:
name: Publish ${{ matrix.title }} report
needs: [resolve_target, kova, source_performance]
if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
actions: read
contents: read
strategy:
fail-fast: false
matrix:
include:
- lane: mock-provider
title: mock provider
- lane: mock-deep-profile
title: mock deep profile
- lane: live-openai-candidate
title: live OpenAI candidate
env:
LANE_ID: ${{ matrix.lane }}
PERFORMANCE_PUBLISHER_HELPER: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-publish-files.mjs
PERFORMANCE_REPORT_SELECTOR: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-selector.mjs
PUBLISHED_REPORT_MAX_FILE_BYTES: "50000000"
REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }}
steps:
- name: Decide report publication lane
id: lane
shell: bash
run: |
set -euo pipefail
run_lane=true
if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
run_lane=false
fi
if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
run_lane=false
fi
echo "run=$run_lane" >> "$GITHUB_OUTPUT"
- name: Checkout performance publisher helper
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.sha }}
path: .artifacts/performance-publisher
sparse-checkout: |
scripts/lib/kova-report-publish-files.mjs
scripts/lib/kova-report-selector.mjs
sparse-checkout-cone-mode: false
fetch-depth: 1
persist-credentials: false
- name: Resolve Kova artifact
id: artifact
if: steps.lane.outputs.run == 'true'
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
prefix="openclaw-performance-${LANE_ID}-${GITHUB_RUN_ID}-"
source_prefix="openclaw-performance-source-${GITHUB_RUN_ID}-"
if ! artifacts="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" --jq '.artifacts[] | select(.expired == false) | [.id, .name] | @tsv')"; then
echo "::${annotation}::Unable to list Kova artifacts for this workflow run."
exit 1
fi
artifact_id=""
producer_attempt=0
source_artifact_id=""
source_producer_attempt=0
while IFS=$'\t' read -r candidate_id candidate_name; do
if [[ "$candidate_name" == "$prefix"* ]]; then
candidate_attempt="${candidate_name#"$prefix"}"
[[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > producer_attempt )); then
artifact_id="$candidate_id"
producer_attempt="$candidate_attempt"
fi
elif [[ "$LANE_ID" == "mock-provider" && "$candidate_name" == "$source_prefix"* ]]; then
candidate_attempt="${candidate_name#"$source_prefix"}"
[[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > source_producer_attempt )); then
source_artifact_id="$candidate_id"
source_producer_attempt="$candidate_attempt"
fi
fi
done <<< "$artifacts"
if [[ -z "$artifact_id" ]]; then
echo "::${annotation}::No Kova artifact is available for ${LANE_ID} through attempt ${GITHUB_RUN_ATTEMPT}."
exit 1
fi
artifact_ids="$artifact_id"
if [[ "$LANE_ID" == "mock-provider" ]]; then
if [[ -z "$source_artifact_id" ]]; then
echo "::${annotation}::No source performance artifact is available through attempt ${GITHUB_RUN_ATTEMPT}."
exit 1
fi
artifact_ids="${artifact_ids},${source_artifact_id}"
if (( source_producer_attempt > producer_attempt )); then
producer_attempt="$source_producer_attempt"
fi
fi
echo "id=$artifact_id" >> "$GITHUB_OUTPUT"
echo "ids=$artifact_ids" >> "$GITHUB_OUTPUT"
echo "producer_attempt=$producer_attempt" >> "$GITHUB_OUTPUT"
echo "source_producer_attempt=$source_producer_attempt" >> "$GITHUB_OUTPUT"
- name: Create isolated publisher paths
id: paths
if: steps.artifact.outputs.id != ''
shell: bash
run: |
set -euo pipefail
input_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX")"
reports_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX")"
chmod 0700 "$input_root" "$reports_root"
echo "input_root=$input_root" >> "$GITHUB_OUTPUT"
echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
- name: Download Kova artifacts
id: download
if: steps.artifact.outputs.id != ''
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ steps.artifact.outputs.ids }}
path: ${{ steps.paths.outputs.input_root }}
- name: Prepare clawgrit report commit
id: prepare
if: steps.download.outcome == 'success'
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
ARTIFACT_ID: ${{ steps.artifact.outputs.id }}
INPUT_ROOT: ${{ steps.paths.outputs.input_root }}
REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }}
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.producer_attempt }}
SOURCE_PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.source_producer_attempt }}
shell: bash
run: |
set -euo pipefail
echo "ready=false" >> "$GITHUB_OUTPUT"
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
input_root="$(realpath "$INPUT_ROOT")"
reports_root="$(realpath "$REPORTS_ROOT")"
if [[ "$input_root" != "$RUNNER_TEMP/"* || "$reports_root" != "$RUNNER_TEMP/"* ]]; then
echo "::${annotation}::Publisher paths escaped RUNNER_TEMP."
exit 1
fi
if [[ ! "$ARTIFACT_ID" =~ ^[0-9]+$ || ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" || ! "$SOURCE_PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$SOURCE_PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then
echo "::${annotation}::Kova artifact attempt metadata is invalid."
exit 1
fi
# download-artifact extracts one ID directly, but nests multiple IDs
# under artifact names. Recursive Kova lookup supports both layouts.
mapfile -d '' report_dirs < <(find "$input_root" -type d -path "*/reports/${LANE_ID}" -print0)
if [[ "${#report_dirs[@]}" != "1" ]]; then
echo "::${annotation}::Expected exactly one Kova report directory for ${LANE_ID}; found ${#report_dirs[@]}."
exit 1
fi
if ! report_json="$(node "$PERFORMANCE_REPORT_SELECTOR" --report-dir "${report_dirs[0]}")"; then
echo "::${annotation}::Exactly one full Kova JSON report is required for ${LANE_ID}."
exit 1
fi
report_json="$(realpath "$report_json")"
if [[ "$report_json" != "$input_root/"* || ! -s "$report_json" ]]; then
echo "::${annotation}::Kova JSON report is outside the downloaded artifact or empty."
exit 1
fi
report_md_path="${report_json%.json}.md"
if [[ -e "$report_md_path" ]]; then
if [[ ! -f "$report_md_path" || -L "$report_md_path" ]]; then
echo "::${annotation}::Kova Markdown report is not a canonical artifact file."
exit 1
fi
report_md="$(realpath "$report_md_path")"
if [[ "$report_md" != "$input_root/"* ]]; then
echo "::${annotation}::Kova Markdown report escaped the downloaded artifact."
exit 1
fi
else
report_md=""
fi
mapfile -d '' summaries < <(find "$input_root" -type f -path "*/summaries/${LANE_ID}.md" -print0)
mapfile -d '' bundles < <(find "$input_root" -type d -path "*/bundles/${LANE_ID}" -print0)
if [[ "${#summaries[@]}" != "1" || "${#bundles[@]}" != "1" ]]; then
echo "::${annotation}::Kova summary or bundle is missing for ${LANE_ID}."
exit 1
fi
summary="$(realpath "${summaries[0]}")"
bundle="$(realpath "${bundles[0]}")"
if [[ "$summary" != "$input_root/"* || "$bundle" != "$input_root/"* || ! -s "$summary" ]]; then
echo "::${annotation}::Kova summary or bundle escaped the downloaded artifact."
exit 1
fi
if find "$bundle" -type l -print -quit | grep -q . || find "$bundle" ! -type d ! -type f -print -quit | grep -q .; then
echo "::${annotation}::Kova bundle contains a symlink or special file."
exit 1
fi
source_dir=""
if [[ "$LANE_ID" == "mock-provider" ]]; then
if [[ "$SOURCE_PRODUCER_ATTEMPT" == "0" ]]; then
echo "::${annotation}::Source probe artifact attempt metadata is missing."
exit 1
fi
# The mock lane downloads two IDs, so its source artifact is nested
# under the immutable artifact name rather than extracted directly.
source_path="${input_root}/openclaw-performance-source-${GITHUB_RUN_ID}-${SOURCE_PRODUCER_ATTEMPT}/${LANE_ID}"
if [[ ! -d "$source_path" ]]; then
echo "::${annotation}::Source probe artifact is missing for ${LANE_ID}."
exit 1
fi
source_dir="$(realpath "$source_path")"
if [[ "$source_dir" != "$input_root/"* ]] || find "$source_dir" -type l -print -quit | grep -q . || find "$source_dir" ! -type d ! -type f -print -quit | grep -q .; then
echo "::${annotation}::Source probe artifact contains an unsafe path."
exit 1
fi
elif [[ "$SOURCE_PRODUCER_ATTEMPT" != "0" ]]; then
echo "::${annotation}::Unexpected source probe artifact attempt for ${LANE_ID}."
exit 1
fi
git -C "$reports_root" init -b main
git -C "$reports_root" config core.hooksPath /dev/null
git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then
echo "::${annotation}::Unable to fetch openclaw/clawgrit-reports before publishing."
exit 1
fi
git -C "$reports_root" checkout -B main FETCH_HEAD
ref_slug="$(printf '%s' "$TESTED_REF" | tr -c 'A-Za-z0-9._-' '-')"
if [[ -z "$ref_slug" || "$ref_slug" == "." || "$ref_slug" == ".." ]]; then
echo "::${annotation}::Tested ref does not produce a safe report path."
exit 1
fi
run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}"
dest_rel="openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
report_url="https://github.com/openclaw/clawgrit-reports/tree/main/${dest_rel}"
if git -C "$reports_root" cat-file -e "HEAD:${dest_rel}/report.json" 2>/dev/null; then
{
echo "### Clawgrit report already published"
echo
echo "- Published report: ${report_url}"
} >> "$GITHUB_STEP_SUMMARY"
echo "already_published=true" >> "$GITHUB_OUTPUT"
echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
echo "ready=true" >> "$GITHUB_OUTPUT"
exit 0
fi
dest="${reports_root}/${dest_rel}"
mkdir -p "$dest"
install -m 0644 "$report_json" "$dest/report.json"
if [[ -n "$report_md" ]]; then
install -m 0644 "$report_md" "$dest/report.md"
fi
install -m 0644 "$summary" "$dest/index.md"
if [[ -n "$source_dir" ]]; then
mkdir -p "$dest/source"
cp -R "$source_dir"/. "$dest/source/"
if [[ -f "$source_dir/index.md" ]]; then
cat >> "$dest/index.md" <<'EOF'
## Source probes
Additional gateway boot, memory, plugin pressure, mock hello-loop, CLI startup, and SQLite state smoke numbers are in [source/index.md](source/index.md).
EOF
fi
fi
artifact_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${ARTIFACT_ID}"
{
echo
echo "## Full diagnostic artifact"
echo
echo "The complete Kova bundle remains in [Actions artifact ${ARTIFACT_ID}](${artifact_url}); its checksum is published under the bundles directory."
} >> "$dest/index.md"
jq -n \
--arg repository "$GITHUB_REPOSITORY" \
--arg ref "$TESTED_REF" \
--arg sha "$TESTED_SHA" \
--arg workflow_ref "$GITHUB_REF_NAME" \
--arg workflow_sha "$GITHUB_SHA" \
--arg workflow "$GITHUB_WORKFLOW" \
--arg run_id "$GITHUB_RUN_ID" \
--arg run_attempt "$PRODUCER_ATTEMPT" \
--arg lane "$LANE_ID" \
--arg path "$dest_rel" \
'{repository:$repository,ref:$ref,sha:$sha,tested_ref:$ref,tested_sha:$sha,workflow_ref:$workflow_ref,workflow_sha:$workflow_sha,workflow:$workflow,run_id:$run_id,run_attempt:$run_attempt,lane:$lane,path:$path}' \
> "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json"
node "$PERFORMANCE_PUBLISHER_HELPER" \
--bundle-dir "$bundle" \
--bundle-destination "$dest/bundles" \
--publish-root "$dest" \
--max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES"
git -C "$reports_root" config user.name "openclaw-performance[bot]"
git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com"
git -C "$reports_root" add -- openclaw-performance
if ! git -C "$reports_root" diff --cached --quiet; then
git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}"
fi
report_commit="$(git -C "$reports_root" rev-parse HEAD)"
echo "already_published=false" >> "$GITHUB_OUTPUT"
echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
echo "report_commit=$report_commit" >> "$GITHUB_OUTPUT"
echo "dest_rel=$dest_rel" >> "$GITHUB_OUTPUT"
echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
echo "ready=true" >> "$GITHUB_OUTPUT"
- name: Create clawgrit reports app token
id: clawgrit_app_token
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
client-id: Iv23liOECG0slfuhz093
private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}
owner: openclaw
repositories: clawgrit-reports
permission-contents: write
- name: Publish to clawgrit reports
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }}
REPORTS_ROOT: ${{ steps.prepare.outputs.reports_root }}
REPORT_COMMIT: ${{ steps.prepare.outputs.report_commit }}
DEST_REL: ${{ steps.prepare.outputs.dest_rel }}
REPORT_URL: ${{ steps.prepare.outputs.report_url }}
shell: bash
run: |
set -euo pipefail
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
if [[ -z "${CLAWGRIT_REPORTS_APP_TOKEN:-}" ]]; then
{
echo "### Clawgrit report publish unavailable"
echo
echo "The repository-scoped ClawSweeper GitHub App token could not be created."
} >> "$GITHUB_STEP_SUMMARY"
echo "::${annotation}::ClawSweeper GitHub App token is unavailable; Kova artifacts were uploaded, but the report cannot be published."
exit 1
fi
token="$CLAWGRIT_REPORTS_APP_TOKEN"
unset CLAWGRIT_REPORTS_APP_TOKEN
auth_header="$(printf 'x-access-token:%s' "$token" | base64 | tr -d '\n')"
unset token
echo "::add-mask::${auth_header}"
reports_root="$(realpath "$REPORTS_ROOT")"
if [[ "$reports_root" != "$RUNNER_TEMP/"* || ! "$REPORT_COMMIT" =~ ^[0-9a-f]{40}$ || ! "$DEST_REL" =~ ^openclaw-performance/[A-Za-z0-9._-]+/[0-9]+-[0-9]+/[A-Za-z0-9._-]+$ ]]; then
echo "::${annotation}::Prepared report metadata failed validation."
exit 1
fi
if [[ "$REPORT_URL" != "https://github.com/openclaw/clawgrit-reports/tree/main/${DEST_REL}" ]]; then
echo "::${annotation}::Prepared report URL failed validation."
exit 1
fi
git_local() {
git -C "$reports_root" -c core.hooksPath=/dev/null "$@"
}
push_report() {
GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null GIT_CONFIG_KEY_1=http.https://github.com/.extraheader GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}" timeout 120s git -C "$reports_root" push origin HEAD:main
}
write_publish_success() {
{
echo "### Clawgrit report published"
echo
echo "- Published report: ${REPORT_URL}"
} >> "$GITHUB_STEP_SUMMARY"
}
write_publish_failure() {
{
echo "### Clawgrit report publish failed"
echo
echo "Kova artifacts were uploaded, but publishing the clawgrit report failed after $1 attempts."
echo "Check the ClawSweeper GitHub App installation and contents permission for openclaw/clawgrit-reports."
} >> "$GITHUB_STEP_SUMMARY"
echo "::${annotation}::Kova artifacts uploaded, but clawgrit report publish failed after $1 attempts."
}
if [[ "$(git_local config --local --get core.hooksPath)" != "/dev/null" || "$(git_local remote get-url origin)" != "https://github.com/openclaw/clawgrit-reports.git" ]]; then
echo "::${annotation}::Prepared reports checkout failed integrity validation."
exit 1
fi
git_local cat-file -e "${REPORT_COMMIT}^{commit}"
report_commit="$REPORT_COMMIT"
for attempt in 1 2 3 4 5; do
if push_report; then
write_publish_success
exit 0
fi
sleep $((attempt * 2))
if ! timeout 120s git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main; then
if [[ "$attempt" != "5" ]]; then
echo "::warning::Unable to refresh openclaw/clawgrit-reports after publish attempt ${attempt}; retrying."
continue
fi
else
if git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json" 2>/dev/null; then
# A timed-out push may have succeeded; verify its unique report before failing.
write_publish_success
exit 0
fi
if [[ "$attempt" != "5" ]]; then
git_local checkout --detach FETCH_HEAD
# The run path is unique; only the shared latest pointer can conflict.
# Replaying with the current commit wins that pointer deterministically.
if ! git_local cherry-pick -X theirs "$report_commit"; then
git_local cherry-pick --abort || true
echo "::${annotation}::Unable to replay the clawgrit report after a concurrent publish."
exit 1
fi
report_commit="$(git_local rev-parse HEAD)"
continue
fi
fi
write_publish_failure "$attempt"
exit 1
done
artifact_only_guard:
name: Verify artifact-only report mode
needs: [resolve_target, kova, publish]
if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.publish_reports != true }}
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
steps:
- name: Verify report publisher stayed disabled
env:
PUBLISH_RESULT: ${{ needs.publish.result }}
shell: bash
run: |
set -euo pipefail
if [[ "$PUBLISH_RESULT" != "skipped" ]]; then
echo "::error::Artifact-only performance mode requires the report publisher to stay skipped; got ${PUBLISH_RESULT}."
exit 1
fi
{
echo "### Artifact-only report mode"
echo
echo "Clawgrit report publication stayed disabled; Kova evidence is available only as workflow artifacts."
} >> "$GITHUB_STEP_SUMMARY"