build_runner: debug memory_blocked_steps assert in ci

.github/workflows/ci.yaml

@@ -12,26 +12,6 @@ permissions:
   # Sets permission policy for `GITHUB_TOKEN`
   contents: read
 jobs:
-  x86_64-linux-debug:
-    timeout-minutes: 540
-    runs-on: [self-hosted, Linux, x86_64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: sh ci/x86_64-linux-debug.sh
-  x86_64-linux-debug-llvm:
-    timeout-minutes: 540
-    runs-on: [self-hosted, Linux, x86_64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: sh ci/x86_64-linux-debug-llvm.sh
   x86_64-linux-release:
     timeout-minutes: 540
     runs-on: [self-hosted, Linux, x86_64]
@@ -42,59 +22,3 @@ jobs:
           fetch-depth: 0
       - name: Build and Test
         run: sh ci/x86_64-linux-release.sh
-  aarch64-linux-debug:
-    runs-on: [self-hosted, Linux, aarch64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: sh ci/aarch64-linux-debug.sh
-  aarch64-linux-release:
-    runs-on: [self-hosted, Linux, aarch64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: sh ci/aarch64-linux-release.sh
-  aarch64-macos-debug:
-    runs-on: [self-hosted, macOS, aarch64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: ci/aarch64-macos-debug.sh
-  aarch64-macos-release:
-    runs-on: [self-hosted, macOS, aarch64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: ci/aarch64-macos-release.sh
-  x86_64-windows-debug:
-    timeout-minutes: 420
-    runs-on: [self-hosted, Windows, x86_64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: ci/x86_64-windows-debug.ps1
-  x86_64-windows-release:
-    timeout-minutes: 420
-    runs-on: [self-hosted, Windows, x86_64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and Test
-        run: ci/x86_64-windows-release.ps1

ci/x86_64-linux-release.sh

@@ -56,6 +56,7 @@ stage3-release/bin/zig build \
 stage3-release/bin/zig build test docs \
   --maxrss 21000000000 \
   -Dlldb=$HOME/deps/lldb-zig/Release-e0a42bb34/bin/lldb \
+  -Dlibc-test-path=$HOME/deps/libc-test-f2bac77 \
   -fqemu \
   -fwasmtime \
   -Dstatic-llvm \

lib/compiler/build_runner.zig

@@ -713,12 +713,18 @@ fn runStepNames(
             if (step.state == .skipped_oom) continue;
 
             thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{
-                &wait_group, b, step, step_prog, run,
+                &wait_group, b, step, step_prog, run, false,
             });
         }
     }
 
-    assert(run.memory_blocked_steps.items.len == 0);
+    // assert(run.memory_blocked_steps.items.len == 0);
+    if (run.memory_blocked_steps.items.len != 0) {
+        std.debug.panic("build terminated with {d} memory blocked steps; first is '{s}'", .{
+            run.memory_blocked_steps.items.len,
+            run.memory_blocked_steps.items[0].name,
+        });
+    }
 
     var test_skip_count: usize = 0;
     var test_fail_count: usize = 0;
@@ -1189,6 +1195,7 @@ fn workerMakeOneStep(
     s: *Step,
     prog_node: std.Progress.Node,
     run: *Run,
+    was_memory_blocked: bool,
 ) void {
     const thread_pool = &run.thread_pool;
 
@@ -1199,11 +1206,13 @@ fn workerMakeOneStep(
         switch (@atomicLoad(Step.State, &dep.state, .seq_cst)) {
             .success, .skipped => continue,
             .failure, .dependency_failure, .skipped_oom => {
+                if (was_memory_blocked) unreachable; // all our dependencies were okay earlier; that shouldn't change!
                 @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst);
                 if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);
                 return;
             },
             .precheck_done, .running => {
+                if (was_memory_blocked) unreachable; // all our dependencies were okay earlier; that shouldn't change!
                 // dependency is not finished yet.
                 return;
             },
@@ -1285,7 +1294,7 @@ fn workerMakeOneStep(
         // Successful completion of a step, so we queue up its dependants as well.
         for (s.dependants.items) |dep| {
             thread_pool.spawnWg(wg, workerMakeOneStep, .{
-                wg, b, dep, prog_node, run,
+                wg, b, dep, prog_node, run, false,
             });
         }
     }
@@ -1310,7 +1319,7 @@ fn workerMakeOneStep(
                 remaining -= dep.max_rss;
 
                 thread_pool.spawnWg(wg, workerMakeOneStep, .{
-                    wg, b, dep, prog_node, run,
+                    wg, b, dep, prog_node, run, true,
                 });
             } else {
                 run.memory_blocked_steps.items[i] = dep;