From 911761007213d4acf289070402f166999aee572c Mon Sep 17 00:00:00 2001 From: "Voight-Kampff (bot)" <151556158+sunholo-voight-kampff@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:02:45 +0200 Subject: [PATCH] =?UTF-8?q?test(ci-flake):=20M2=20=E2=80=94=20migrate=20ca?= =?UTF-8?q?ll=20sites=20to=20the=20testutil=20gate,=20add=20deterministic?= =?UTF-8?q?=20httptest=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes AC1, AC2, AC3'(a/b/c), AC4, AC6, AC7 of m-ci-flake-systemic-fix. Executed by codex gpt-5.6-sol; all gates re-run by the controller OUTSIDE the sandbox (in-sandbox verdicts are uninformative — workspace-write denies loopback binds, which is indistinguishable from a real regression). The central change: AILANG's live-network tests no longer run by default. pre-sprint : --- FAIL: TestNetHttpPost/httpPost_to_httpbin.org (0.37s) net_test.go:380: got: 503 Service Temporarily Unavailable post-M2 : --- SKIP: TestNetHttpPost/httpPost_to_httpbin.org --- PASS: TestNetHttpPost/local_success_response --- PASS: TestNetHttpPost/local_non-2xx_response That 503 was measured first-party on dev during this iteration, with no proxy env set — i.e. #588/#561 reproducing live. CI was green only because the old Getenv("CI") opt-out hid it there, which is the defect class this sprint removes. Changes (12 test files, 8 packages, +86/-115; no production code, no workflows): - internal/pkg/gitcache_test.go, internal/effects/net_test.go — inert gates and the SKIP_NET_TESTS/CI/GITHUB_ACTIONS opt-out replaced by testutil.RequiresLiveNetwork - internal/effects/net_test.go — live non-2xx now tolerated-and-logged, and NEW deterministic httptest coverage for netHTTPPost success + non-2xx. Requires ctx.Net.AllowHTTP and ctx.Net.AllowLocalhost (both default false; an httptest URL is http://127.0.0.1:, blocked twice over) - cmd/ailang/main_test.go — runCLI, buildAilang and runAilangBin routed through testutil.RunBounded (120s cap), including the go-run helper the design doc missed - cmd/ailang/main_run_pipe_test.go — eventOneBudget removed, minGap untouched - internal/eval_harness/reference_solutions_test.go — per-language warm-up run (the actual #587 fix; the 60s constant was only the symptom) - 6 further inert gates removed, each guarded body re-read in context first - internal/coordinator/provider_script_test.go — allowlisted with a reason: it gates Unix shell signal semantics, not network, so the network helper would misdescribe it Verified outside the sandbox: full suite unpoisoned rc=0 106 ok 0 FAIL full suite poisoned [AC3c] rc=0 106 ok 0 FAIL (pre-sprint: rc=1, 1 FAIL) poisoned ./internal/pkg/ rc=0 (pre-sprint: rc=1, git clone exit 128; unpoisoned control rc=0, so the poison is the mechanism) AILANG_LIVE_NET=1 [AC4] --- PASS (1.55s, a real clone — not a skip) live lane + poison [AC11] FATAL "live network lane is misconfigured" gofmt 0 files, go vet rc=0 grep sweeps: testing.Short() 7->0 files, Getenv(CI|GHA) 2->1 (allowlisted), eventOneBudget 4->0, minGap 3 (control, untouched), 60*time.Second 1->0, warm 0->4 Mutation-tested by the controller, negative control green first, each mutation proven landed by sha256 + byte-diff, both reverts byte-identical: - remove the live gate -> SKIP becomes PASS (subtest runs live again) - remove AllowLocalhost -> both deterministic subtests FAIL NOTE for M4/M5 wiring: AC3'(b) is NOT rc-checkable. The first mutation left the package rc=0 because httpbin happened to answer that time. AC3'(b) must assert on the literal "--- SKIP:" lines, never on the exit code. Two plan discrepancies found and reported by the executor, both confirmed: - `go build ./...` is not a usable gate here: it fails identically on unmodified dev (cmd/wasm and gen/main have no native main). Not caused by M2. - PR #532's sync.Once builder was ALREADY on dev via #564 (3c28cc322, merged 2026-08-03), so the "#532 collides with M2" blocker was already moot. #532 closed as superseded. Co-Authored-By: codex Co-Authored-By: Claude Opus 4.8 --- cmd/ailang/main_run_pipe_test.go | 22 +------ cmd/ailang/main_test.go | 53 ++++------------ cmd/ailang/serve_api_mcp_surface_test.go | 3 - internal/ai/ollama/client_test.go | 4 -- internal/coordinator/provider_script_test.go | 4 ++ internal/effects/net_test.go | 60 +++++++++++++++---- internal/effects/process_test.go | 3 - .../eval_harness/reference_solutions_test.go | 27 +++++---- .../gen/golang/contracts_integration_test.go | 9 --- internal/pipeline/validate_effects_test.go | 4 -- internal/pkg/gitcache_test.go | 6 +- internal/pkg/publish_validator_test.go | 6 -- 12 files changed, 86 insertions(+), 115 deletions(-) diff --git a/cmd/ailang/main_run_pipe_test.go b/cmd/ailang/main_run_pipe_test.go index b0aa0cbbe..a8054e02e 100644 --- a/cmd/ailang/main_run_pipe_test.go +++ b/cmd/ailang/main_run_pipe_test.go @@ -8,7 +8,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "strings" "testing" "time" @@ -105,7 +104,6 @@ export func main() -> () ! {IO, Clock} { // If the buffer is broken (events only flush at exit), they all arrive // at ~the same time near the end (~1.5-2s). gotByEvent := map[string]time.Duration{} - deadline := time.After(4 * time.Second) collect: for len(gotByEvent) < 3 { select { @@ -114,7 +112,7 @@ collect: break collect } gotByEvent[ev.line] = ev.at - case <-deadline: + case <-ctx.Done(): break collect } } @@ -140,24 +138,6 @@ collect: gap, minGap) } - // Belt-and-suspenders: also assert EVENT_1 arrived before total runtime - // elapsed (i.e. before all three sleeps would have completed sequentially). - // - // On Windows the ailang binary cold-start cost is ~1.7s vs <0.5s on - // Linux/macOS — runner-VM filesystem + process-launch overhead — so the - // budget is widened there. The load-bearing assertion is the gap check - // above (EVENT_1 → EVENT_2 ≥ 200ms); this check is redundant guardrail. - eventOneBudget := 1500 * time.Millisecond - if runtime.GOOS == "windows" { - eventOneBudget = 3500 * time.Millisecond - } - if gotByEvent["EVENT_1"] > eventOneBudget { - t.Errorf("EVENT_1 arrived at %s — too late (budget %s). Expected first println "+ - "to appear before the program had time to call all three sleeps. "+ - "Suggests stdout is buffered until exit.", - gotByEvent["EVENT_1"], eventOneBudget) - } - // Diagnostic output for debugging. t.Logf("event timings: EVENT_1=%s, EVENT_2=%s, EVENT_3=%s", gotByEvent["EVENT_1"], diff --git a/cmd/ailang/main_test.go b/cmd/ailang/main_test.go index 0decb85c8..98b799fcd 100644 --- a/cmd/ailang/main_test.go +++ b/cmd/ailang/main_test.go @@ -1,16 +1,17 @@ package main import ( - "bytes" + "fmt" "os" - "os/exec" "path/filepath" "runtime" "strings" "sync" "testing" + "time" "github.com/sunholo-data/ailang/internal/effects" + "github.com/sunholo-data/ailang/internal/testutil" ) // runCLI runs the ailang CLI with given arguments and returns stdout, stderr, and exit code @@ -23,26 +24,7 @@ func runCLI(t *testing.T, args ...string) (stdout, stderr string, exitCode int) t.Fatalf("Failed to get project root: %v", err) } - cmd := exec.Command("go", append([]string{"run", "./cmd/ailang"}, args...)...) - cmd.Dir = projectRoot // Run from project root so paths resolve correctly - - var outBuf, errBuf bytes.Buffer - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - - err = cmd.Run() - stdout = outBuf.String() - stderr = errBuf.String() - - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - exitCode = exitErr.ExitCode() - } else { - t.Fatalf("Failed to run CLI: %v", err) - } - } - - return stdout, stderr, exitCode + return testutil.RunBounded(t, projectRoot, 120*time.Second, "go", append([]string{"run", "./cmd/ailang"}, args...)...) } func TestCLI_Version(t *testing.T) { @@ -428,7 +410,7 @@ var ( ailangBinOnce sync.Once ailangBinPath string ailangBinErr error - ailangBinOutput []byte + ailangBinOutput string ) // buildAilang builds the ailang binary once per test run and returns its path. @@ -463,9 +445,11 @@ func buildAilang(t *testing.T) string { return } ailangBinPath = filepath.Join(dir, binName) - cmd := exec.Command("go", "build", "-o", ailangBinPath, "./cmd/ailang") - cmd.Dir = projectRoot - ailangBinOutput, ailangBinErr = cmd.CombinedOutput() + stdout, stderr, exitCode := testutil.RunBounded(t, projectRoot, 120*time.Second, "go", "build", "-o", ailangBinPath, "./cmd/ailang") + ailangBinOutput = stdout + stderr + if exitCode != 0 { + ailangBinErr = fmt.Errorf("go build exited with code %d", exitCode) + } }) if ailangBinErr != nil { t.Fatalf("Failed to build ailang: %v\n%s", ailangBinErr, ailangBinOutput) @@ -480,22 +464,7 @@ func runAilangBin(t *testing.T, binPath string, args ...string) (stdout, stderr if err != nil { t.Fatalf("Failed to get project root: %v", err) } - cmd := exec.Command(binPath, args...) - cmd.Dir = projectRoot - var outBuf, errBuf bytes.Buffer - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - err = cmd.Run() - stdout = outBuf.String() - stderr = errBuf.String() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - exitCode = exitErr.ExitCode() - } else { - t.Fatalf("Failed to run ailang: %v", err) - } - } - return stdout, stderr, exitCode + return testutil.RunBounded(t, projectRoot, 60*time.Second, binPath, args...) } func TestCLI_Exit_Code0(t *testing.T) { diff --git a/cmd/ailang/serve_api_mcp_surface_test.go b/cmd/ailang/serve_api_mcp_surface_test.go index 705b05817..7e920e35e 100644 --- a/cmd/ailang/serve_api_mcp_surface_test.go +++ b/cmd/ailang/serve_api_mcp_surface_test.go @@ -14,9 +14,6 @@ import ( ) func TestServeAPI_MCPToolSurface(t *testing.T) { - if testing.Short() { - t.Skip("builds and drives the serve-api stdio MCP binary") - } binary := buildAilang(t) moduleRoot := t.TempDir() diff --git a/internal/ai/ollama/client_test.go b/internal/ai/ollama/client_test.go index 52f47ca62..fd478908d 100644 --- a/internal/ai/ollama/client_test.go +++ b/internal/ai/ollama/client_test.go @@ -93,10 +93,6 @@ func TestGuessProvider(t *testing.T) { // TestCheckConnection is an integration test that requires Ollama running. // Skip if Ollama is not available. func TestCheckConnection(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") - } - client, err := NewClient() if err != nil { t.Fatalf("NewClient() error = %v", err) diff --git a/internal/coordinator/provider_script_test.go b/internal/coordinator/provider_script_test.go index 98cbb8934..1bc2fa9b5 100644 --- a/internal/coordinator/provider_script_test.go +++ b/internal/coordinator/provider_script_test.go @@ -250,6 +250,10 @@ func TestScriptProvider_Execute(t *testing.T) { }) t.Run("timeout kills script", func(t *testing.T) { + // Gatelint R2 allowlist reason: this is a Unix shell/grandchild signal + // semantics test, not a live-network test, so the network opt-in helper + // would misstate its requirement. CI runners skip the known flaky shell + // behavior while local Unix runs retain coverage. // Skip in CI - exec.CommandContext signal handling is unreliable // on Linux when using "bash -c" because the shell doesn't forward // signals to child processes. This causes the test to be flaky. diff --git a/internal/effects/net_test.go b/internal/effects/net_test.go index d0f9e6b3c..4545b08b3 100644 --- a/internal/effects/net_test.go +++ b/internal/effects/net_test.go @@ -2,11 +2,13 @@ package effects import ( "net" - "os" + "net/http" + "net/http/httptest" "strings" "testing" "github.com/sunholo-data/ailang/internal/eval" + "github.com/sunholo-data/ailang/internal/testutil" ) // TestValidateIP_MetadataServer tests the cloud metadata server exception. @@ -359,10 +361,7 @@ func TestNetHttpPost(t *testing.T) { }) t.Run("httpPost to httpbin.org", func(t *testing.T) { - // Skip in CI environments due to unreliable external network access - if os.Getenv("SKIP_NET_TESTS") != "" || os.Getenv("CI") != "" || os.Getenv("GITHUB_ACTIONS") != "" { - t.Skip("Skipping network test in CI environment (unreliable external access)") - } + testutil.RequiresLiveNetwork(t) url := &eval.StringValue{Value: "https://httpbin.org/post"} body := &eval.StringValue{Value: `{"test": "data", "value": 42}`} @@ -377,13 +376,57 @@ func TestNetHttpPost(t *testing.T) { if !ok { t.Errorf("Expected StringValue, got %T", result) } else if !strings.Contains(strResult.Value, "httpbin.org") { - t.Errorf("Expected response containing 'httpbin.org', got: %s", strResult.Value) + t.Logf("Live endpoint returned a non-canonical response (possibly non-2xx); deterministic response assertions are covered by local-server subtests: %s", strResult.Value) } } } else { t.Logf("Network error (expected in some environments): %v", err) } }) + + for _, tc := range []struct { + name string + statusCode int + response string + }{ + {name: "local success response", statusCode: http.StatusOK, response: `{"ok":true}`}, + {name: "local non-2xx response", statusCode: http.StatusServiceUnavailable, response: `{"error":"unavailable"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + w.WriteHeader(tc.statusCode) + _, _ = w.Write([]byte(tc.response)) + })) + defer server.Close() + + localCtx := NewEffContext([]string{}) + localCtx.Grant(NewCapability("Net")) + localCtx.Net = NewNetContext() + // Local test servers use plain HTTP on loopback, so both capabilities + // must be explicit. This deterministic coverage therefore exercises a + // different capability posture from the HTTPS live-endpoint subtest. + localCtx.Net.AllowHTTP = true + localCtx.Net.AllowLocalhost = true + + result, err := netHTTPPost(localCtx, []eval.Value{ + &eval.StringValue{Value: server.URL}, + &eval.StringValue{Value: `{"request":"body"}`}, + }) + if err != nil { + t.Fatalf("netHTTPPost: %v", err) + } + got, ok := result.(*eval.StringValue) + if !ok { + t.Fatalf("result type = %T, want *eval.StringValue", result) + } + if got.Value != tc.response { + t.Errorf("response = %q, want %q", got.Value, tc.response) + } + }) + } } // TestNetBodySizeLimit verifies response size limiting @@ -395,10 +438,7 @@ func TestNetBodySizeLimit(t *testing.T) { t.Run("small response under limit", func(t *testing.T) { // httpbin.org/get returns ~270 bytes, should exceed 100 byte limit - // Skip in CI environments due to unreliable external network access - if os.Getenv("SKIP_NET_TESTS") != "" || os.Getenv("CI") != "" || os.Getenv("GITHUB_ACTIONS") != "" { - t.Skip("Skipping network test in CI environment (unreliable external access)") - } + testutil.RequiresLiveNetwork(t) url := &eval.StringValue{Value: "https://httpbin.org/get"} _, err := netHTTPGet(ctx, []eval.Value{url}) diff --git a/internal/effects/process_test.go b/internal/effects/process_test.go index 4727ddc8e..751367808 100644 --- a/internal/effects/process_test.go +++ b/internal/effects/process_test.go @@ -434,9 +434,6 @@ func TestProcessExec_WaitDelay_OrphanGrandchildNoHang(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("bash test requires unix") } - if testing.Short() { - t.Skip("WaitDelay timing test (~5s)") - } ctx := newProcessCtx() args := []eval.Value{ &eval.StringValue{Value: "bash"}, diff --git a/internal/eval_harness/reference_solutions_test.go b/internal/eval_harness/reference_solutions_test.go index 6d6beb1da..cf64e0fdb 100644 --- a/internal/eval_harness/reference_solutions_test.go +++ b/internal/eval_harness/reference_solutions_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" "time" + + "github.com/sunholo-data/ailang/internal/testutil" ) // referenceSolution describes one benchmark/language pair. @@ -67,6 +69,20 @@ func testReferenceSolutions(t *testing.T, lang string) { t.Skipf("cannot find repo root: %v", err) } + // Pay each language runtime's cold-start cost once outside the asserted + // cases. The warm-up result is intentionally ignored: these tests assert + // the checked-in reference programs, not runtime startup behavior. + warmupCode := `console.log("warmup")` + if lang == "go" { + warmupCode = "package main\nimport \"fmt\"\nfunc main() { fmt.Println(\"warmup\") }\n" + } + for _, rs := range referenceSolutionsTable { + if rs.lang == lang { + _, _ = rs.runner().Run(warmupCode, testutil.HangGuard(t, 120*time.Second)) + break + } + } + for _, rs := range referenceSolutionsTable { if rs.lang != lang { continue @@ -79,17 +95,8 @@ func testReferenceSolutions(t *testing.T, lang string) { t.Fatalf("reference solution not found: %s: %v", srcPath, err) } - // Reference solutions are tiny programs; wall-clock is dominated - // by interpreter startup (~slow on Windows CI runners — node alone - // can take >20s cold). recursion_fibonacci was getting the only - // 60s slot, but fizzbuzz hits the same 30s cliff on Windows. Give - // every benchmark the same generous slot — the only thing the - // shorter limit was buying was faster failure on a hang, and any - // real hang would still time out well before 60s of useful work. - timeout := 60 * time.Second - runner := rs.runner() - result, err := runner.Run(string(code), timeout) + result, err := runner.Run(string(code), testutil.HangGuard(t, 120*time.Second)) if err != nil { t.Fatalf("runner error: %v", err) } diff --git a/internal/gen/golang/contracts_integration_test.go b/internal/gen/golang/contracts_integration_test.go index 9c7e12367..908882b39 100644 --- a/internal/gen/golang/contracts_integration_test.go +++ b/internal/gen/golang/contracts_integration_test.go @@ -16,11 +16,6 @@ import ( // 3. Runs tests that trigger contract violations // 4. Verifies panics occur with correct messages func TestContractViolation_Integration(t *testing.T) { - // Skip in short mode (these tests compile and run Go code) - if testing.Short() { - t.Skip("skipping integration test in short mode") - } - // Create temp directory for test tmpDir, err := os.MkdirTemp("", "contract_test_*") if err != nil { @@ -246,10 +241,6 @@ func TestIncrement_EnsuresViolation(t *testing.T) { // TestContractViolation_NoVerify verifies that without --verify-contracts, // contract violations do NOT cause panics (contracts are just comments) func TestContractViolation_NoVerify(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") - } - tmpDir, err := os.MkdirTemp("", "contract_noverify_*") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) diff --git a/internal/pipeline/validate_effects_test.go b/internal/pipeline/validate_effects_test.go index 8e827550b..1429704ce 100644 --- a/internal/pipeline/validate_effects_test.go +++ b/internal/pipeline/validate_effects_test.go @@ -52,10 +52,6 @@ func TestValidateEffects_LargeArrayPerformance(t *testing.T) { // TestValidateEffects_LinearScaling verifies that effect checking scales linearly with input size. // Note: This test uses warmup iterations and takes minimum times to be robust on CI. func TestValidateEffects_LinearScaling(t *testing.T) { - if testing.Short() { - t.Skip("skipping performance test in short mode") - } - sizes := []int{10, 50, 100} var times []time.Duration const iterations = 5 // Run multiple times and take minimum diff --git a/internal/pkg/gitcache_test.go b/internal/pkg/gitcache_test.go index 1c1dea257..b4fb70972 100644 --- a/internal/pkg/gitcache_test.go +++ b/internal/pkg/gitcache_test.go @@ -4,6 +4,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/sunholo-data/ailang/internal/testutil" ) func TestGitCache_CacheDir_Deterministic(t *testing.T) { @@ -46,9 +48,7 @@ func TestGitCache_Resolve_RequiresTagOrRev(t *testing.T) { // Integration test — requires git and network access func TestGitCache_Resolve_RealRepo(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") - } + testutil.RequiresLiveNetwork(t) cache := &GitCache{baseDir: t.TempDir()} diff --git a/internal/pkg/publish_validator_test.go b/internal/pkg/publish_validator_test.go index 63e8ee663..10262712c 100644 --- a/internal/pkg/publish_validator_test.go +++ b/internal/pkg/publish_validator_test.go @@ -106,9 +106,6 @@ export func main() -> () ! {FS} = // TestRunSmokeInTempDir_Timeout asserts that a smoke that runs longer than // the timeout is killed and reported as TimedOut. func TestRunSmokeInTempDir_Timeout(t *testing.T) { - if testing.Short() { - t.Skip("skipping timeout test in -short mode") - } bin := findAilangBinary(t) pkgDir := t.TempDir() @@ -178,9 +175,6 @@ export func main() -> () ! {FS, IO} = // and the runner honours it. Pairs with TestRunSmokeInTempDir_Timeout which // exercises the default 30s case. func TestRunSmokeInTempDir_RespectsCustomTimeout(t *testing.T) { - if testing.Short() { - t.Skip("skipping timeout test in -short mode") - } bin := findAilangBinary(t) pkgDir := t.TempDir()