Skip to content

Commit cdccbcc

Browse files
rhussrhuss
committed
feat: add warm pool experiment scripts, manifests, and RFC scaffold
Add measurement scripts for cold-start baseline, warm pool claims, readiness gate patterns, sidecar readiness, env var injection, and combined best-case scenarios. Each script produces CSV output with per-phase timestamps for latency analysis. Includes: - 6 measurement scripts with shared library functions - K8s manifests for SandboxTemplate, WarmPool, Claims, and DaemonSet - Go readiness sidecar with signal-file-based health endpoint - RFC template document with structured sections for results - Deep review findings from 5-agent code review Assisted-By: 🤖 Claude Code Signed-off-by: Roland Huß <rhuss@redhat.com>
1 parent 73adeb3 commit cdccbcc

23 files changed

Lines changed: 2563 additions & 19 deletions

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,9 @@ scripts/lint-mermaid/node_modules/
230230
/result
231231
/result-*
232232

233+
# Warm pool experiment shared libraries (overrides Python lib/ pattern above)
234+
!experiments/lib/
235+
233236
# spex: generated/local files (only constitution is committed)
234237
# Use worktree-specific pattern to avoid shadowing root .claude/ tracked files
235238
.claude/worktrees/*/.claude/

‎experiments/lib/common.sh‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
#!/usr/bin/env bash
2+
# Shared measurement functions for warm pool experiments.
3+
# Source this file from experiment scripts: source "$(dirname "$0")/../lib/common.sh"
4+
5+
set -euo pipefail
6+
7+
EXPERIMENT_ID="${EXPERIMENT_ID:-unknown}"
8+
RESULTS_DIR="${RESULTS_DIR:-experiments/results}"
9+
10+
CLEANUP_RESOURCES=()
11+
register_cleanup() { CLEANUP_RESOURCES+=("$1"); }
12+
deregister_cleanup() { CLEANUP_RESOURCES=("${CLEANUP_RESOURCES[@]/$1}"); }
13+
_cleanup_on_exit() {
14+
for res in "${CLEANUP_RESOURCES[@]}"; do
15+
[[ -z "$res" ]] && continue
16+
log "Cleaning up: $res"
17+
kubectl delete $res --ignore-not-found --wait=false 2>/dev/null || true
18+
done
19+
}
20+
trap _cleanup_on_exit EXIT
21+
22+
log() {
23+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
24+
}
25+
26+
ensure_results_dir() {
27+
mkdir -p "$RESULTS_DIR"
28+
}
29+
30+
capture_timestamp() {
31+
if command -v gdate &>/dev/null; then
32+
gdate +%s%N
33+
elif date +%s%N | grep -qv N; then
34+
date +%s%N
35+
else
36+
# Fallback: seconds with 000000000 appended (no nanosecond support)
37+
echo "$(date +%s)000000000"
38+
fi
39+
}
40+
41+
write_csv_header() {
42+
local file="$1"
43+
echo "run,experiment,config,create_ts,ready_ts,delta_ms,pod,status,scheduled_ms,pulled_ms,init_ms,supervisor_ms,ssh_ms" > "$file"
44+
}
45+
46+
write_csv_row() {
47+
local file="$1"
48+
local run="$2"
49+
local config="$3"
50+
local create_ts="$4"
51+
local ready_ts="$5"
52+
local delta_ms="$6"
53+
local pod="$7"
54+
local row_status="$8"
55+
local scheduled_ms="${9:-}"
56+
local pulled_ms="${10:-}"
57+
local init_ms="${11:-}"
58+
local supervisor_ms="${12:-}"
59+
local ssh_ms="${13:-}"
60+
echo "${run},${EXPERIMENT_ID},${config},${create_ts},${ready_ts},${delta_ms},${pod},${row_status},${scheduled_ms},${pulled_ms},${init_ms},${supervisor_ms},${ssh_ms}" >> "$file"
61+
}
62+
63+
_iso_to_epoch() {
64+
local ts="$1"
65+
if [[ -z "$ts" || "$ts" == "null" ]]; then
66+
echo ""
67+
return
68+
fi
69+
if command -v gdate &>/dev/null; then
70+
gdate -d "$ts" +%s 2>/dev/null || echo ""
71+
else
72+
date -d "$ts" +%s 2>/dev/null || echo ""
73+
fi
74+
}
75+
76+
_event_ts() {
77+
local events_json="$1"
78+
local reason="$2"
79+
local pick="${3:-first}"
80+
local selector=".reason == \"$reason\""
81+
if [[ "$reason" == *"|"* ]]; then
82+
local r1="${reason%%|*}"
83+
local r2="${reason#*|}"
84+
selector=".reason == \"$r1\" or .reason == \"$r2\""
85+
fi
86+
if [[ "$pick" == "last" ]]; then
87+
echo "$events_json" | jq -r "[.items[] | select($selector)] | last | (.lastTimestamp // .eventTime // null)" 2>/dev/null
88+
else
89+
echo "$events_json" | jq -r "[.items[] | select($selector)] | first | (.lastTimestamp // .eventTime // null)" 2>/dev/null
90+
fi
91+
}
92+
93+
extract_phase_deltas() {
94+
local events_json="$1"
95+
local create_ts="$2"
96+
local create_s=$((create_ts / 1000000000))
97+
98+
local sched_ts pull_ts init_ts start_ts
99+
sched_ts=$(_event_ts "$events_json" "Scheduled" "first")
100+
pull_ts=$(_event_ts "$events_json" "Pulled|Pulling" "last")
101+
init_ts=$(_event_ts "$events_json" "Created" "first")
102+
start_ts=$(_event_ts "$events_json" "Started" "last")
103+
104+
local sched_ms="" pull_ms="" init_ms="" start_ms=""
105+
106+
local epoch
107+
epoch=$(_iso_to_epoch "$sched_ts")
108+
[[ -n "$epoch" ]] && sched_ms=$(( (epoch - create_s) * 1000 ))
109+
110+
epoch=$(_iso_to_epoch "$pull_ts")
111+
[[ -n "$epoch" ]] && pull_ms=$(( (epoch - create_s) * 1000 ))
112+
113+
epoch=$(_iso_to_epoch "$init_ts")
114+
[[ -n "$epoch" ]] && init_ms=$(( (epoch - create_s) * 1000 ))
115+
116+
epoch=$(_iso_to_epoch "$start_ts")
117+
[[ -n "$epoch" ]] && start_ms=$(( (epoch - create_s) * 1000 ))
118+
119+
echo "${sched_ms},${pull_ms},${init_ms},${start_ms},"
120+
}
121+
122+
collect_pod_events() {
123+
local pod="$1"
124+
local ns="${2:-default}"
125+
kubectl get events \
126+
--namespace="$ns" \
127+
--field-selector="involvedObject.name=$pod,involvedObject.kind=Pod" \
128+
--sort-by='.lastTimestamp' \
129+
-o json 2>/dev/null || echo '{"items":[]}'
130+
}
131+
132+
detect_adoption() {
133+
local pod_name="$1"
134+
local ns="${2:-default}"
135+
if [[ -z "$pod_name" ]]; then
136+
echo "unknown"
137+
return
138+
fi
139+
140+
local events_json
141+
events_json=$(collect_pod_events "$pod_name" "$ns")
142+
143+
local scheduled_count
144+
scheduled_count=$(echo "$events_json" | \
145+
jq '[.items[] | select(.reason == "Scheduled")] | length' 2>/dev/null || echo "0")
146+
147+
local pulled_count
148+
pulled_count=$(echo "$events_json" | \
149+
jq '[.items[] | select(.reason == "Pulling" or .reason == "Pulled")] | length' 2>/dev/null || echo "0")
150+
151+
if (( scheduled_count == 0 && pulled_count == 0 )); then
152+
echo "warm-adopted"
153+
else
154+
echo "cold-fallback"
155+
fi
156+
}
157+
158+
compute_stats() {
159+
local csv_file="$1"
160+
if [[ ! -f "$csv_file" ]]; then
161+
log "ERROR: CSV file not found: $csv_file"
162+
return 1
163+
fi
164+
165+
awk -F',' '
166+
NR == 1 { next }
167+
$8 != "ok" { skipped++; next }
168+
$6 == "" { skipped++; next }
169+
{
170+
v = $6 + 0
171+
n++
172+
vals[n] = v
173+
sum += v
174+
if (n == 1 || v < min) min = v
175+
if (n == 1 || v > max) max = v
176+
}
177+
END {
178+
if (n == 0) {
179+
print "No data rows found."
180+
exit 1
181+
}
182+
183+
# Sort values (insertion sort)
184+
for (i = 2; i <= n; i++) {
185+
key = vals[i]
186+
j = i - 1
187+
while (j >= 1 && vals[j] > key) {
188+
vals[j+1] = vals[j]
189+
j--
190+
}
191+
vals[j+1] = key
192+
}
193+
194+
p50_idx = int(n * 0.5 + 0.5)
195+
p90_idx = int(n * 0.9 + 0.5)
196+
if (p50_idx < 1) p50_idx = 1
197+
if (p90_idx < 1) p90_idx = 1
198+
if (p50_idx > n) p50_idx = n
199+
if (p90_idx > n) p90_idx = n
200+
201+
mean = sum / n
202+
printf "\n--- Statistics (%d samples) ---\n", n
203+
printf " min: %10.1f ms\n", min
204+
printf " max: %10.1f ms\n", max
205+
printf " mean: %10.1f ms\n", mean
206+
printf " p50: %10.1f ms\n", vals[p50_idx]
207+
printf " p90: %10.1f ms\n", vals[p90_idx]
208+
printf "-------------------------------\n"
209+
if (skipped > 0) printf " (%d non-ok rows excluded)\n", skipped
210+
}
211+
' "$csv_file"
212+
}

‎experiments/lib/wait-ready.sh‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env bash
2+
# Pod and resource readiness wait functions.
3+
# Source this file from experiment scripts: source "$(dirname "$0")/../lib/wait-ready.sh"
4+
5+
set -euo pipefail
6+
7+
wait_for_ready() {
8+
local resource="$1"
9+
local timeout_s="${2:-120}"
10+
kubectl wait --for=condition=Ready "$resource" --timeout="${timeout_s}s" 2>/dev/null
11+
}
12+
13+
wait_for_pod_ready() {
14+
local pod="$1"
15+
local ns="${2:-default}"
16+
local timeout_s="${3:-120}"
17+
local elapsed=0
18+
19+
while [[ $elapsed -lt $timeout_s ]]; do
20+
local status
21+
status=$(kubectl get pod "$pod" \
22+
--namespace="$ns" \
23+
-o jsonpath='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}' \
24+
2>/dev/null) || true
25+
26+
if [[ "$status" == "True" ]]; then
27+
return 0
28+
fi
29+
30+
local phase
31+
phase=$(kubectl get pod "$pod" \
32+
--namespace="$ns" \
33+
-o jsonpath='{.status.phase}' \
34+
2>/dev/null) || true
35+
36+
if [[ "$phase" == "Failed" || "$phase" == "Succeeded" ]]; then
37+
echo "Pod $pod terminated with phase: $phase" >&2
38+
return 1
39+
fi
40+
41+
if (( elapsed % 10 == 0 && elapsed > 0 )); then
42+
echo "Waiting for pod $pod to be Ready... (${elapsed}s/${timeout_s}s, phase: ${phase:-unknown})" >&2
43+
fi
44+
45+
sleep 1
46+
elapsed=$((elapsed + 1))
47+
done
48+
49+
echo "Timeout: pod $pod not Ready after ${timeout_s}s" >&2
50+
return 1
51+
}
52+
53+
wait_for_sandbox_ready() {
54+
local name="$1"
55+
local ns="${2:-default}"
56+
local timeout_s="${3:-120}"
57+
local elapsed=0
58+
59+
while [[ $elapsed -lt $timeout_s ]]; do
60+
local status
61+
status=$(kubectl get sandbox "$name" \
62+
--namespace="$ns" \
63+
-o jsonpath='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}' \
64+
2>/dev/null) || true
65+
66+
if [[ "$status" == "True" ]]; then
67+
return 0
68+
fi
69+
70+
if (( elapsed % 10 == 0 && elapsed > 0 )); then
71+
echo "Waiting for sandbox $name to be Ready... (${elapsed}s/${timeout_s}s)" >&2
72+
fi
73+
74+
sleep 1
75+
elapsed=$((elapsed + 1))
76+
done
77+
78+
echo "Timeout: sandbox $name not Ready after ${timeout_s}s" >&2
79+
return 1
80+
}

‎experiments/manifests/image-prepull-daemonset.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: image-prepull
5+
namespace: default
6+
labels:
7+
app.kubernetes.io/name: image-prepull
8+
app.kubernetes.io/part-of: openshell-warm-pool
9+
spec:
10+
selector:
11+
matchLabels:
12+
app.kubernetes.io/name: image-prepull
13+
template:
14+
metadata:
15+
labels:
16+
app.kubernetes.io/name: image-prepull
17+
spec:
18+
initContainers:
19+
- name: pull-sandbox
20+
image: ghcr.io/nvidia/openshell/sandbox:latest
21+
imagePullPolicy: IfNotPresent
22+
command: ["/bin/true"]
23+
containers:
24+
- name: pause
25+
image: registry.k8s.io/pause:3.10
26+
resources:
27+
requests:
28+
cpu: "1m"
29+
memory: "4Mi"
30+
limits:
31+
cpu: "1m"
32+
memory: "4Mi"
33+
tolerations:
34+
- operator: Exists

‎experiments/manifests/readiness-gate-pod.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: readiness-gate-test-PLACEHOLDER
5+
spec:
6+
readinessGates:
7+
- conditionType: sandbox.openshell.io/claimed
8+
containers:
9+
- name: sandbox
10+
image: ghcr.io/nvidia/openshell/sandbox:latest
11+
ports:
12+
- containerPort: 2222
13+
resources:
14+
requests:
15+
cpu: "500m"
16+
memory: "512Mi"

‎experiments/manifests/sandbox-claim.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: extensions.agents.x-k8s.io/v1beta1
2+
kind: SandboxClaim
3+
metadata:
4+
name: claim-PLACEHOLDER
5+
spec:
6+
warmPoolRef:
7+
name: openshell-warm-pool
8+
9+
# Variant with env var injection (requires envVarsInjectionPolicy: Allowed on template):
10+
# spec:
11+
# warmPoolRef:
12+
# name: openshell-warm-pool
13+
# env:
14+
# - name: AGENT_ID
15+
# value: "agent-001"
16+
# - name: SESSION_TOKEN
17+
# value: "tok-xxx"

‎experiments/manifests/sandbox-template.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
apiVersion: extensions.agents.x-k8s.io/v1beta1
2+
kind: SandboxTemplate
3+
metadata:
4+
name: openshell-warm
5+
spec:
6+
podTemplateSpec:
7+
spec:
8+
containers:
9+
- name: sandbox
10+
image: ghcr.io/nvidia/openshell/sandbox:latest
11+
ports:
12+
- containerPort: 2222
13+
name: ssh
14+
readinessProbe:
15+
tcpSocket:
16+
port: 2222
17+
initialDelaySeconds: 2
18+
periodSeconds: 10
19+
resources:
20+
requests:
21+
cpu: "500m"
22+
memory: "512Mi"
23+
limits:
24+
cpu: "2"
25+
memory: "2Gi"

0 commit comments

Comments
 (0)