Skip to content

Commit c8a4475

Browse files
actions-useractions-user
committed
ci: re-apply ops overlay after hard-reset to upstream/main
1 parent 22793a1 commit c8a4475

6 files changed

Lines changed: 1484 additions & 0 deletions

File tree

‎.github/workflows/build-from-source.yml‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
# UNIVERSAL build-from-source workflow — BYTE-IDENTICAL across every Model B fork.
2+
#
3+
# DO NOT customize this file per-fork. If a fork needs different behavior, use
4+
# GitHub Actions repository Variables (Settings → Secrets and variables →
5+
# Actions → Variables):
6+
#
7+
# DOCKERFILE override auto-detected path (default: root Dockerfile, then shallowest)
8+
# BUILD_CONTEXT override docker build context (default: . — repo root)
9+
# PLATFORMS override platform list (default: linux/amd64,linux/arm64)
10+
# BUILD_ARGS multi-line KEY=VALUE pairs passed to docker build (default: none).
11+
# Used by forks whose upstream Dockerfile declares an ARG with no
12+
# default that the upstream's own CI passes externally (e.g.
13+
# tika-docker's TIKA_VERSION via republish-images.sh).
14+
# IS_SOURCE_BUILT job-level gate: build runs only when set to 'true'. Plain-mirror
15+
# forks set 'false' so the workflow file stays installed (uniform
16+
# scaffolding) but the job no-ops; fork-publish.yml owns :latest.
17+
# FREE_DISK_SPACE opt-in (set 'true') for forks whose images blow past the
18+
# 14GB default disk on ubuntu-latest (vllm-class CUDA images).
19+
# Adds ~3 min per build; reclaims ~30GB by removing pre-installed
20+
# Android SDK / .NET / Haskell / large npm caches.
21+
# RUNS_ON per-fork runner-label override (default: "ubuntu-latest").
22+
# JSON-encoded — single string for one label, or JSON array for
23+
# multiple. Set to '["self-hosted","lan-docker","big-build"]'
24+
# for forks that exceed hosted-runner capacity (vllm).
25+
# BUILD_TIMEOUT_MINUTES per-fork job-timeout override (default 360, the
26+
# GitHub-hosted-runner ceiling). Raise as a ONE-SHOT bootstrap
27+
# on forks whose first cold-cache build exceeds 360min (vllm
28+
# single-arch CUDA: ~5-6h first run, ~30-45m once :buildcache
29+
# is populated). Revert to default after first green run.
30+
# Honest-fact #97.
31+
# PRECOMPILED_WHEEL_BASE_URL master switch for the upstream-prebuilt-wheel
32+
# build path. Set to e.g. `https://wheels.vllm.ai` on forks
33+
# whose upstream publishes per-commit prebuilt wheels AND
34+
# whose carry stack is 100% non-compiled (Python/MD only).
35+
# When set, the build passes ${PREFIX}_USE_PRECOMPILED=1 + the
36+
# resolved commit + variant as build-args, and the fork's
37+
# Dockerfile carry can early-out the local C++/CUDA compile.
38+
# When unset (default for all forks), build path is unchanged.
39+
# sync-upstream.yml writes PRECOMPILED_WHEEL_COMMIT after
40+
# curl-verifying the wheel exists; if no variant in
41+
# PRECOMPILED_WHEEL_VARIANT_CHAIN publishes, sync CLEARS
42+
# PRECOMPILED_WHEEL_COMMIT to empty so the build falls back
43+
# to source compile rather than serving stale-ABI binaries
44+
# (the only safe semantics — a wheel built against SHA-N-1
45+
# with Python carries against SHA-N's APIs segfaults at
46+
# runtime with no build-time signal). Honest-fact #99.
47+
# PRECOMPILED_WHEEL_COMMIT SHA of the upstream commit whose wheel to fetch.
48+
# Managed by sync-upstream — operator should NOT set this
49+
# directly UNLESS PRECOMPILED_WHEEL_COMMIT_FROZEN=true (then
50+
# sync respects the manual pin). Empty when no wheel is
51+
# currently available; build falls back to source compile.
52+
# PRECOMPILED_WHEEL_VARIANT wheel build variant (e.g. `cu130`, `cu128`).
53+
# Operator-set initial value; sync-upstream updates in-place
54+
# when the variant chain falls through to a different one
55+
# (unless PRECOMPILED_WHEEL_COMMIT_FROZEN=true).
56+
# PRECOMPILED_WHEEL_VARIANT_CHAIN optional. Space-separated fallback list
57+
# (e.g. `cu130 cu128`). Sync-upstream tries each in order;
58+
# first one whose wheel exists wins. Defaults to the single
59+
# PRECOMPILED_WHEEL_VARIANT value.
60+
# PRECOMPILED_WHEEL_BUILD_ARG_PREFIX prefix for the synthesized build-arg
61+
# env names. Default `VLLM` — synthesizes VLLM_USE_PRECOMPILED,
62+
# VLLM_PRECOMPILED_WHEEL_COMMIT, etc. For forks whose upstream
63+
# uses a different convention (MinerU, torchao, future), set
64+
# to that project's prefix (e.g. `MINERU`) → MINERU_USE_PRECOMPILED.
65+
# Keeps the template generic across vendor opt-ins.
66+
# PRECOMPILED_WHEEL_COMMIT_FROZEN set to `true` to pin
67+
# PRECOMPILED_WHEEL_COMMIT/_VARIANT manually — sync-upstream
68+
# will NOT overwrite them when set. Use for testing a specific
69+
# wheel SHA, or for rolling back from a bad upstream commit.
70+
# Default unset (sync auto-manages).
71+
#
72+
# If a fork's Dockerfile needs out-of-context pre-build steps, the right fix is
73+
# to cherry-pick a multi-stage self-contained Dockerfile from upstream into our
74+
# intarweb-dev — NOT to add pre-build logic here.
75+
#
76+
# Codified in oss-contributing:ghcr-fork-mirror skill honest-fact #52.
77+
78+
name: Build from source → GHCR
79+
80+
on:
81+
push:
82+
branches: [intarweb-dev]
83+
workflow_dispatch:
84+
85+
# Single-flight per repo: schedule-driven sync push + Heal K dispatch + manual
86+
# workflow_dispatch can all converge. cancel-in-progress: false because builds
87+
# are expensive (vllm 30-45 min, bifrost 45 min) — let the in-flight one finish,
88+
# queue the next behind it. Honest-fact #74.
89+
concurrency:
90+
group: build-from-source-${{ github.repository }}
91+
cancel-in-progress: false
92+
93+
permissions:
94+
contents: read
95+
packages: write
96+
97+
jobs:
98+
build:
99+
# IS_SOURCE_BUILT Variable is the authoritative source-vs-mirror switch
100+
# for the fleet. Setting it to anything other than 'true' (or leaving it
101+
# unset) skips the build entirely. This keeps the WORKFLOW FILE itself
102+
# byte-identical across every fork (uniform infra scaffolding) while
103+
# giving operators a per-fork Variable to control source-build vs
104+
# plain-mirror behavior without removing files or disabling workflows.
105+
# Plain-mirror forks (docker-neo4j etc) still have build-from-source.yml
106+
# installed but the job no-ops; fork-publish.yml owns :latest for them.
107+
#
108+
# Branch gate: when triggered by a push, only run on intarweb-dev. The
109+
# `on.push.branches: [intarweb-dev]` filter above is supposed to handle
110+
# this, but in practice GitHub still records 0-second false-failure runs
111+
# against build-from-source.yml when sync-upstream's bot pushes to main
112+
# (empty jobs[], shows red in Actions UI, head_branch=main, name=workflow
113+
# path rather than display name). Belt-and-suspenders branch gate here
114+
# ensures the job no-ops cleanly to a green "skipped" rather than a
115+
# mysterious red failure if the trigger ever evaluates against main /
116+
# master / any other ref. Manual workflow_dispatch and workflow_call
117+
# paths remain unaffected. Honest-fact #109.
118+
if: >-
119+
vars.IS_SOURCE_BUILT == 'true'
120+
&& (
121+
github.event_name == 'workflow_dispatch'
122+
|| github.event_name == 'workflow_call'
123+
|| (github.event_name == 'push' && github.ref == 'refs/heads/intarweb-dev')
124+
)
125+
# Per-fork override via repo Variable RUNS_ON (default: ubuntu-latest hosted).
126+
# Set to '["self-hosted","lan-docker","big-build"]' (JSON array, single line)
127+
# for forks whose builds exceed hosted-runner capacity (16GB RAM / 14GB disk
128+
# — vllm's CUDA csrc-build is the canonical case). Honest-fact #89.
129+
runs-on: ${{ fromJSON(vars.RUNS_ON || '"ubuntu-latest"') }}
130+
# Per-fork override via repo Variable BUILD_TIMEOUT_MINUTES (default 360, the
131+
# GitHub-hosted-runner hard ceiling). Raise to 720 as a ONE-SHOT bootstrap on
132+
# forks whose first build needs to populate :buildcache from scratch (vllm
133+
# at single-arch CUDA: ~5-6h cold-cache vs ~30-45m warm). Revert to 360 once
134+
# warm. Codified after 27340784996 / 27373307060 cancelled at 360min on the
135+
# vllm fork's first cold-cache run. Honest-fact #91.
136+
timeout-minutes: ${{ fromJSON(vars.BUILD_TIMEOUT_MINUTES || '360') }}
137+
# Read-once capture of precompiled-wheel Variables at job start. Subsequent
138+
# steps reference $PRECOMPILED_* env vars (resolved once at job-init) rather
139+
# than ${{ vars.* }} (re-resolved per-step expression). This makes an
140+
# in-flight build immune to mid-job Variable mutations from a concurrent
141+
# sync-upstream writing PRECOMPILED_WHEEL_COMMIT/_VARIANT. Eliminates the
142+
# read/write race architect-review #1 flagged. Honest-fact #99.
143+
env:
144+
PRECOMPILED_BASE_URL: ${{ vars.PRECOMPILED_WHEEL_BASE_URL }}
145+
PRECOMPILED_COMMIT: ${{ vars.PRECOMPILED_WHEEL_COMMIT }}
146+
PRECOMPILED_VARIANT: ${{ vars.PRECOMPILED_WHEEL_VARIANT }}
147+
PRECOMPILED_PREFIX: ${{ vars.PRECOMPILED_WHEEL_BUILD_ARG_PREFIX || 'VLLM' }}
148+
OPERATOR_BUILD_ARGS: ${{ vars.BUILD_ARGS }}
149+
steps:
150+
- uses: actions/checkout@v4
151+
with:
152+
persist-credentials: false
153+
154+
# Optional disk reclamation for forks whose images blow past the 14GB
155+
# default disk on ubuntu-latest hosted runners (vllm being the canonical
156+
# offender — its CUDA build artifacts + intermediate layers exceed 20GB
157+
# easily). Frees ~30GB by removing pre-installed Android SDK, .NET,
158+
# Haskell, large npm caches, and codeql databases. Costs ~3 min — gated
159+
# behind FREE_DISK_SPACE Variable so non-vllm-class forks don't pay it.
160+
- name: 🧹 Free disk space (opt-in via FREE_DISK_SPACE Variable)
161+
if: vars.FREE_DISK_SPACE == 'true'
162+
uses: jlumbroso/free-disk-space@v1.3.1
163+
with:
164+
tool-cache: true
165+
android: true
166+
dotnet: true
167+
haskell: true
168+
large-packages: true
169+
docker-images: false
170+
swap-storage: false
171+
172+
- name: 🔍 Resolve Dockerfile
173+
id: df
174+
env:
175+
DOCKERFILE_VAR: ${{ vars.DOCKERFILE }}
176+
run: |
177+
set -euo pipefail
178+
# 1. Explicit override (repo Variable) always wins.
179+
if [ -n "${DOCKERFILE_VAR}" ]; then
180+
echo " ✓ using DOCKERFILE override: ${DOCKERFILE_VAR}"
181+
echo "path=${DOCKERFILE_VAR}" >> "$GITHUB_OUTPUT"
182+
echo "skip=false" >> "$GITHUB_OUTPUT"
183+
exit 0
184+
fi
185+
# 2. Prefer a root Dockerfile.
186+
if [ -f Dockerfile ]; then
187+
echo " ✓ found root Dockerfile"
188+
echo "path=Dockerfile" >> "$GITHUB_OUTPUT"
189+
echo "skip=false" >> "$GITHUB_OUTPUT"
190+
exit 0
191+
fi
192+
# 3. Deterministic discovery: shallowest non-test Dockerfile.
193+
cand="$(find . -maxdepth 4 -type f -name Dockerfile \
194+
-not -path './node_modules/*' \
195+
-not -path './.git/*' \
196+
-not -path '*/test/*' \
197+
-not -path '*/tests/*' \
198+
-not -path '*/example*/*' \
199+
-not -path './vendor/*' \
200+
-not -path './third_party/*' \
201+
| sed 's|^\./||' \
202+
| awk '{print gsub(/\//,"/"), $0}' \
203+
| sort -n | head -n1 | cut -d' ' -f2- || true)"
204+
if [ -z "$cand" ]; then
205+
echo " ✗ no Dockerfile found → plain-mirror fallback (no image built)"
206+
echo "skip=true" >> "$GITHUB_OUTPUT"
207+
exit 0
208+
fi
209+
echo " ✓ auto-found: $cand"
210+
echo "path=$cand" >> "$GITHUB_OUTPUT"
211+
echo "skip=false" >> "$GITHUB_OUTPUT"
212+
213+
- name: Set up QEMU
214+
if: steps.df.outputs.skip != 'true'
215+
uses: docker/setup-qemu-action@v3
216+
217+
- name: Set up Buildx
218+
if: steps.df.outputs.skip != 'true'
219+
uses: docker/setup-buildx-action@v3
220+
221+
- name: Log in to ghcr.io
222+
if: steps.df.outputs.skip != 'true'
223+
uses: docker/login-action@v3
224+
with:
225+
registry: ghcr.io
226+
username: ${{ github.actor }}
227+
password: ${{ secrets.GITHUB_TOKEN }}
228+
229+
- name: 🔡 Compute image (lowercase — GHCR requires lowercase repo names)
230+
id: imgname
231+
run: |
232+
# vars.IMAGE_NAME may already be lowercase (set as such manually), but
233+
# the github.event.repository.name fallback can be CamelCase if the fork
234+
# repo name has uppercase letters (e.g. RetroSaveManager). docker/build-push
235+
# rejects ANY uppercase in the image path. fork-publish.yml does the same
236+
# via tr; we do it here. Honest-fact #63.
237+
NAME="${{ vars.IMAGE_NAME || github.event.repository.name }}"
238+
NAME_LOWER=$(echo "$NAME" | tr '[:upper:]' '[:lower:]')
239+
echo "image=ghcr.io/${{ github.repository_owner }}/${NAME_LOWER}" >> "$GITHUB_OUTPUT"
240+
241+
- name: 🏷️ Docker metadata
242+
id: meta
243+
if: steps.df.outputs.skip != 'true'
244+
uses: docker/metadata-action@v5
245+
with:
246+
# Per-fork override via repo Variable IMAGE_NAME — for forks where the
247+
# published image name differs from the repo name (docker-autoheal repo →
248+
# autoheal image). Falls back to repo name. Same as fork-publish.yml so
249+
# ONE image per fork. Lowercased above (metadata-action's lowercase only
250+
# applies to tags, not to the images input — uppercase repo names slip
251+
# through and break build-push with "repository name must be lowercase").
252+
images: ${{ steps.imgname.outputs.image }}
253+
tags: |
254+
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/intarweb-dev' }}
255+
type=sha,prefix=sha-,format=short
256+
257+
- name: 🧩 Synthesize build-args (BUILD_ARGS + optional precompiled-wheel envs)
258+
# Combines operator-set BUILD_ARGS with auto-derived precompiled-wheel
259+
# build-args when PRECOMPILED_WHEEL_BASE_URL is set on the fork. For
260+
# forks WITHOUT that Variable (the default), output is byte-identical to
261+
# the pre-existing `build-args: ${{ vars.BUILD_ARGS || '' }}` behavior.
262+
# Uses $PRECOMPILED_* env vars captured at job start (job-level env
263+
# block) — immune to mid-build Variable mutations. Synthesizes envs with
264+
# the configured PREFIX (default `VLLM`), so non-vllm forks adopting this
265+
# pattern (MinerU, torchao, future) just set
266+
# PRECOMPILED_WHEEL_BUILD_ARG_PREFIX without template surgery.
267+
# Honest-fact #99/#104.
268+
id: bargs
269+
if: steps.df.outputs.skip != 'true'
270+
run: |
271+
set -euo pipefail
272+
BARGS="${OPERATOR_BUILD_ARGS:-}"
273+
if [ -n "${PRECOMPILED_BASE_URL:-}" ] && [ -n "${PRECOMPILED_COMMIT:-}" ]; then
274+
# Build the precompiled-wheel envs via printf so the multi-line
275+
# KEY=VALUE block doesn't dedent the surrounding YAML run block
276+
# (honest-fact #104 — broken-yaml-from-bash-multiline-assignment
277+
# 0-second-failed all 22 forks 2026-06-12).
278+
#
279+
# Emit BOTH ${PREFIX}_PRECOMPILED_WHEEL_COMMIT and
280+
# ${PREFIX}_MERGE_BASE_COMMIT to the same SHA — different
281+
# Dockerfile carries may key off either name (vllm-bot's carry
282+
# uses MERGE_BASE_COMMIT; theoretical future carries may use
283+
# the more-explicit PRECOMPILED_WHEEL_COMMIT). Both work.
284+
# Honest-fact #105.
285+
PREFIX="${PRECOMPILED_PREFIX:-VLLM}"
286+
EXTRA=$(printf '%s_USE_PRECOMPILED=1\n%s_PRECOMPILED_WHEEL_COMMIT=%s\n%s_MERGE_BASE_COMMIT=%s\n%s_PRECOMPILED_WHEEL_VARIANT=%s\n%s_PRECOMPILED_WHEEL_BASE_URL=%s' "$PREFIX" "$PREFIX" "$PRECOMPILED_COMMIT" "$PREFIX" "$PRECOMPILED_COMMIT" "$PREFIX" "$PRECOMPILED_VARIANT" "$PREFIX" "$PRECOMPILED_BASE_URL")
287+
if [ -n "$BARGS" ]; then
288+
BARGS="${BARGS}"$'\n'"${EXTRA}"
289+
else
290+
BARGS="${EXTRA}"
291+
fi
292+
echo " ✓ precompiled-wheel build-args appended (prefix=${PREFIX} commit=${PRECOMPILED_COMMIT} variant=${PRECOMPILED_VARIANT})"
293+
elif [ -n "${PRECOMPILED_BASE_URL:-}" ] && [ -z "${PRECOMPILED_COMMIT:-}" ]; then
294+
# Master switch on, but no resolved commit. Sync-upstream cleared it
295+
# because no wheel published for the current upstream SHA. Build will
296+
# fall back to source compile — degraded perf, but correct binaries
297+
# (vs the silent-ABI-break risk of serving a stale wheel).
298+
echo "::warning::PRECOMPILED_WHEEL_BASE_URL set but PRECOMPILED_WHEEL_COMMIT empty — sync-upstream couldn't find a wheel for current upstream SHA. Falling back to source compile."
299+
fi
300+
{
301+
echo "build_args<<__EOB__"
302+
echo "$BARGS"
303+
echo "__EOB__"
304+
} >> "$GITHUB_OUTPUT"
305+
306+
- name: 🚀 Build & push
307+
if: steps.df.outputs.skip != 'true'
308+
uses: docker/build-push-action@v6
309+
with:
310+
context: ${{ vars.BUILD_CONTEXT || '.' }}
311+
file: ${{ steps.df.outputs.path }}
312+
platforms: ${{ vars.PLATFORMS || 'linux/amd64,linux/arm64' }}
313+
# build-args synthesized above. Combines operator-set BUILD_ARGS with
314+
# auto-derived precompiled-wheel envs when PRECOMPILED_WHEEL_BASE_URL
315+
# is set. For forks without it, equivalent to BUILD_ARGS alone.
316+
# Honest-facts #88, #99.
317+
build-args: ${{ steps.bargs.outputs.build_args }}
318+
# BuildKit registry-cache. Reuses prior build's intermediate layers
319+
# across runs to dramatically cut wall time on warm builds (vllm:
320+
# ~30min → ~10-12min target). cache-from is read-only — first run on
321+
# a fresh fork misses everything (cold cache, no penalty). cache-to
322+
# `mode=max` exports ALL intermediate stages (not just the final
323+
# image), so the next run can resume mid-stage. `compression=zstd`
324+
# for faster decompression on cache pulls.
325+
# ignore-error=true on cache-to: if the push to :buildcache fails
326+
# (perms, network), the build still succeeds; just warm cache misses
327+
# on next run. Honest-fact #107.
328+
cache-from: type=registry,ref=${{ steps.imgname.outputs.image }}:buildcache
329+
cache-to: type=registry,ref=${{ steps.imgname.outputs.image }}:buildcache,mode=max,compression=zstd,ignore-error=true
330+
push: true
331+
tags: ${{ steps.meta.outputs.tags }}
332+
labels: ${{ steps.meta.outputs.labels }}
333+
provenance: false

0 commit comments

Comments
 (0)