Attrition QA #1592
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Attrition QA | |
| # Runs after every deploy to verify NodeBench quality | |
| on: | |
| # After Vercel deploy completes | |
| deployment_status: | |
| # Manual trigger | |
| workflow_dispatch: | |
| inputs: | |
| api_url: | |
| description: 'NodeBench API URL' | |
| default: 'https://scratchnode.live' | |
| jobs: | |
| qa-crawl: | |
| name: Surface crawl | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' || github.event.deployment_status.state == 'success' | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Crawl all surfaces | |
| run: | | |
| BASE="${{ github.event.inputs.api_url || 'https://scratchnode.live' }}" | |
| if [[ "$BASE" == https://vercel.com/* ]]; then | |
| BASE="https://scratchnode.live" | |
| fi | |
| BASE="${BASE%/}" | |
| echo "Crawling $BASE" | |
| SURFACES=("ask" "library" "connect" "telemetry") | |
| ERRORS=0 | |
| for surface in "${SURFACES[@]}"; do | |
| STATUS=$(curl -L -sS --retry 2 --retry-delay 2 --connect-timeout 10 -o /dev/null -w "%{http_code}" "$BASE/?surface=$surface" --max-time 30) | |
| if [ "$STATUS" = "200" ]; then | |
| echo "PASS $surface: $STATUS" | |
| else | |
| echo "FAIL $surface: $STATUS" | |
| ERRORS=$((ERRORS + 1)) | |
| fi | |
| done | |
| # Check pipeline health | |
| PIPELINE=$(curl -L -sS --retry 1 --connect-timeout 10 "$BASE/api/pipeline/health" --max-time 30 2>/dev/null || echo '{}') | |
| echo "Pipeline health: $PIPELINE" | |
| # Check HyperLoop stats | |
| HYPERLOOP=$(curl -L -sS --retry 1 --connect-timeout 10 "$BASE/api/hyperloop/stats" --max-time 30 2>/dev/null || echo '{}') | |
| echo "HyperLoop stats: $HYPERLOOP" | |
| if [ $ERRORS -gt 0 ]; then | |
| echo "::error::$ERRORS surface(s) failed health check" | |
| exit 1 | |
| fi | |
| golden-queries: | |
| name: Pipeline quality benchmark | |
| runs-on: ubuntu-latest | |
| needs: qa-crawl | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| package-manager-cache: false | |
| node-version: '20' | |
| - name: Install dependencies | |
| run: npm ci --ignore-scripts --no-audit --no-fund || npm install --ignore-scripts --no-audit --no-fund | |
| - name: Run golden queries | |
| env: | |
| NODEBENCH_API_URL: ${{ github.event.inputs.api_url || 'https://scratchnode.live' }} | |
| run: | | |
| if [[ "$NODEBENCH_API_URL" == https://vercel.com/* ]]; then | |
| export NODEBENCH_API_URL="https://scratchnode.live" | |
| fi | |
| npx tsx scripts/attrition/run-golden-queries.ts | |
| continue-on-error: true | |
| - name: Upload results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: golden-query-results | |
| path: scripts/attrition/golden-results.json | |
| - name: Comment results on PR (if applicable) | |
| if: github.event.pull_request | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const results = JSON.parse(fs.readFileSync('scripts/attrition/golden-results.json', 'utf8')); | |
| const body = `## NodeBench Quality Benchmark | |
| **Pass rate:** ${results.passed}/${results.total} (${results.passRate}%) | |
| **Avg confidence:** ${results.avgConfidence}% | |
| **Avg latency:** ${results.avgLatency}ms | |
| ${results.results.filter(r => !r.passed).map(r => | |
| `❌ **${r.id}**: ${r.failures.join(', ')}` | |
| ).join('\n')} | |
| ${results.results.filter(r => r.passed).map(r => | |
| `✅ **${r.id}**: ${r.confidence}% conf, ${r.signals} signals, ${r.sources} sources` | |
| ).join('\n')} | |
| `; | |
| if (context.issue?.number) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: body, | |
| }); | |
| } |