Skip to content

Non-agentic evals: parameterize evalchemy sbatch paths by $USER #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mrinal-bespoke wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Non-agentic evals: parameterize evalchemy sbatch paths by $USER #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
da55d6a
Non-agentic evals: parameterize evalchemy sbatch paths by $USER (shar…
mrinal-bespoke Jul 29, 2026
450e145
Non-agentic evals: parameterize qwen3_eval.sbatch by $USER (missed in…
mrinal-bespoke Jul 29, 2026
38f286a
evalchemy sbatch: conda fallback must validate the ENV exists, not ju…
mrinal-bespoke Jul 29, 2026
3e74bc2
evalchemy sbatch: document that the $USER --output dir must pre-exist
mrinal-bespoke Aug 4, 2026
dc3108d
evalchemy: resolve the repo root by probe, not by assuming a canonica…
mrinal-bespoke Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 57 additions & 7 deletions eval/evalchemy/delphi_eval.sbatch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,14 @@
#SBATCH --gres=gpu:4
#SBATCH --time=08:00:00
#SBATCH --job-name=delphi-eval
#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log
#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log
#
# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT.
# SLURM opens the log file before the job script runs and does NOT create intermediate
# directories -- a missing dir fails the job instantly with "Unable to open file", which
# looks like a launch bug rather than a missing mkdir. It already exists for the original
# operator; a NEW operator runs this once:
# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval
#
# ==============================================================================
# delphi_eval.sbatch — THIN delphi wrapper around evalchemy_eval.sbatch (#6279).
Expand Down Expand Up @@ -33,15 +40,58 @@ set -euo pipefail
# so BASH_SOURCE[0]/dirname resolves to the spool dir (where evalchemy_eval.sbatch does NOT exist) —
# that produced a 4-sec ExitCode-127 "No such file or directory". Pin to the repo clone instead,
# falling back to the BASH_SOURCE dir only for interactive/non-SLURM invocation.
REPO_EVAL_DIR="/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/eval/evalchemy"
if [[ -f "${REPO_EVAL_DIR}/evalchemy_eval.sbatch" ]]; then
SELF_DIR="${REPO_EVAL_DIR}"
else
SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# $USER-scoped so a second operator runs THEIR checkout, not another user's. Resolves to the
# original bfeuer00 paths byte-identically when USER=bfeuer00.
#
# Resolution is by PROBE, not by assumption: try each candidate and take the first that actually
# contains the marker file. A bare "$WORK/$USER/code/OpenThoughts-Agent" default only works for
# operators who cloned at exactly that path; anyone else fell through to the BASH_SOURCE branch,
# which under SLURM is the /var/spool/slurmd/jobNNN/ copy — i.e. it re-created the very
# ExitCode-127 "No such file or directory" this block was written to prevent. It also silently
# mis-pointed CHAT_TEMPLATE_OVERRIDE below (and THINK_TEMPLATE in delphi_rl_think_eval.sbatch).
_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}"
_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00"
_OTA_MARKER="eval/evalchemy/evalchemy_eval.sbatch"
# Walk up from a directory looking for the marker: SLURM_SUBMIT_DIR is wherever the operator ran
# sbatch, which is often a subdirectory of the repo rather than its root.
_ota_up() {
local d="${1:-}"
[ -n "$d" ] || return 1
while [ -n "$d" ] && [ "$d" != "/" ]; do
[ -f "$d/${_OTA_MARKER}" ] && { printf '%s' "$d"; return 0; }
d="$(dirname "$d")"
done
return 1
}
_ota_first_repo() {
local c
for c in "$@"; do
[ -n "$c" ] && [ -f "$c/${_OTA_MARKER}" ] && { printf '%s' "$c"; return 0; }
done
return 1
}
_OTA_REPO="$(_ota_first_repo \
"${OTAGENT_REPO_DIR:-}" \
"${_OTA_WORK}/code/OpenThoughts-Agent" \
"$(_ota_up "${SLURM_SUBMIT_DIR:-}" || true)" \
"${_OTA_SHARED}/code/OpenThoughts-Agent" \
"$(_ota_up "$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || true)" || true)" \
|| true)"
if [ -z "${_OTA_REPO}" ]; then
echo "ERROR: cannot locate an OpenThoughts-Agent checkout containing ${_OTA_MARKER}." >&2
echo " tried, in order:" >&2
echo " OTAGENT_REPO_DIR = ${OTAGENT_REPO_DIR:-<unset>}" >&2
echo " \$USER checkout = ${_OTA_WORK}/code/OpenThoughts-Agent" >&2
echo " SLURM_SUBMIT_DIR = ${SLURM_SUBMIT_DIR:-<unset>} (and parents)" >&2
echo " shared checkout = ${_OTA_SHARED}/code/OpenThoughts-Agent" >&2
echo " script dir = $(dirname "${BASH_SOURCE[0]}") (and parents)" >&2
echo " Set OTAGENT_REPO_DIR=/path/to/OpenThoughts-Agent and resubmit." >&2
exit 1
fi
SELF_DIR="${_OTA_REPO}/eval/evalchemy"

# Delphi-specific defaults (overridable from the environment).
export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/chat_templates/delphi_v0.jinja2}"
export CHAT_TEMPLATE_OVERRIDE="${CHAT_TEMPLATE_OVERRIDE:-${_OTA_REPO}/chat_templates/delphi_v0.jinja2}"
export MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
export MAX_GEN_TOKS="${MAX_GEN_TOKS:-3584}"
export JOB_PREFIX="${JOB_PREFIX:-delphi-eval}"
Expand Down
73 changes: 64 additions & 9 deletions eval/evalchemy/delphi_rl_think_eval.sbatch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,14 @@
#SBATCH --gres=gpu:4
#SBATCH --time=08:00:00
#SBATCH --job-name=delphi-think-eval
#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log
#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log
#
# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT.
# SLURM opens the log file before the job script runs and does NOT create intermediate
# directories -- a missing dir fails the job instantly with "Unable to open file", which
# looks like a launch bug rather than a missing mkdir. It already exists for the original
# operator; a NEW operator runs this once:
# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval
#
# Held-out math eval for the 4 FORCED-THINKING RL grid cells (consolidated local HF dirs).
# IDENTICAL harness to eval/evalchemy/evalchemy_eval.sbatch (same tasks, seeds, gen_kwargs,
Expand All @@ -21,31 +28,79 @@
# Usage: sbatch --job-name=delphi-think-eval-<RUN> delphi_rl_think_eval.sbatch <LOCAL_MODEL_DIR> <RUN_NAME>
set -euo pipefail

# ----- $USER-scoped paths (multi-operator) -----------------------------------------------------
# Resolves to the original bfeuer00 strings byte-identically when USER=bfeuer00. Read-only shared
# runtime (conda, evalchemy-marin clone) falls back to the bfeuer00 copy; WRITE paths are per-user.
_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}"
_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00"
_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; }
# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not
# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with
# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root.
_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; }
_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}"
# Repo root by PROBE, not assumption — see the long note in delphi_eval.sbatch. A bare
# "$WORK/$USER/code/OpenThoughts-Agent" default silently mis-points THINK_TEMPLATE below for any
# operator who cloned elsewhere, and lm-eval then fails on a missing chat template rather than
# saying the repo was not found.
_OTA_MARKER="eval/evalchemy/evalchemy_eval.sbatch"
_ota_up() {
local d="${1:-}"
[ -n "$d" ] || return 1
while [ -n "$d" ] && [ "$d" != "/" ]; do
[ -f "$d/${_OTA_MARKER}" ] && { printf '%s' "$d"; return 0; }
d="$(dirname "$d")"
done
return 1
}
_ota_first_repo() {
local c
for c in "$@"; do
[ -n "$c" ] && [ -f "$c/${_OTA_MARKER}" ] && { printf '%s' "$c"; return 0; }
done
return 1
}
_OTA_REPO="$(_ota_first_repo \
"${OTAGENT_REPO_DIR:-}" \
"${_OTA_WORK}/code/OpenThoughts-Agent" \
"$(_ota_up "${SLURM_SUBMIT_DIR:-}" || true)" \
"${_OTA_SHARED}/code/OpenThoughts-Agent" \
"$(_ota_up "$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || true)" || true)" \
|| true)"
if [ -z "${_OTA_REPO}" ]; then
echo "ERROR: cannot locate an OpenThoughts-Agent checkout containing ${_OTA_MARKER}." >&2
echo " Set OTAGENT_REPO_DIR=/path/to/OpenThoughts-Agent and resubmit." >&2
echo " tried: OTAGENT_REPO_DIR=${OTAGENT_REPO_DIR:-<unset>}, ${_OTA_WORK}/code/OpenThoughts-Agent," >&2
echo " SLURM_SUBMIT_DIR=${SLURM_SUBMIT_DIR:-<unset>} (and parents), ${_OTA_SHARED}/code/OpenThoughts-Agent" >&2
exit 1
fi


# Clean any propagated python env that can shadow conda's base-python stdlib
# (intermittent "ImportError: cannot import name 'JSONEncoder' from 'json'" on some compute nodes).
unset PYTHONHOME PYTHONPATH PYTHONSTARTUP 2>/dev/null || true
source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh
source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh"
# Activate with a small retry — the JSONEncoder import flake is transient (GPFS/conda).
for try in 1 2 3 4 5; do
conda activate evalchemy-marin 2>/dev/null || true
if python -c "from json import JSONEncoder; import sys; assert 'evalchemy-marin' in sys.executable" 2>/dev/null; then
echo ">>> conda evalchemy-marin active (try $try): $(which python)"; break
fi
echo ">>> conda activate flake on try $try; retrying in 10s"; conda deactivate 2>/dev/null || true; sleep 10
source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh
source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh"
done
python -c "from json import JSONEncoder; import sys; assert 'evalchemy-marin' in sys.executable" \
|| { echo "FATAL: could not activate evalchemy-marin cleanly after retries"; exit 17; }
cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin
cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}"

source ~/secrets.env
export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub
export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub
export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}"
export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}"
export HF_HUB_OFFLINE=1
export HF_ALLOW_CODE_EVAL=1
export TOKENIZERS_PARALLELISM=false

CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache
CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}"
mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local}
export HOME="$CACHE_ROOT/hf_local"
export XDG_CACHE_HOME="$CACHE_ROOT/xdg"
Expand All @@ -57,8 +112,8 @@ mkdir -p "$HOME/.cache/flashinfer"

MODEL_DIR="${1:?usage: sbatch ... <LOCAL_MODEL_DIR> <RUN_NAME>}"
RUN_NAME="${2:?usage: sbatch ... <LOCAL_MODEL_DIR> <RUN_NAME>}"
THINK_TEMPLATE="${THINK_TEMPLATE:-/leonardo_work/AIFAC_5C0_290/bfeuer00/code/OpenThoughts-Agent/chat_templates/delphi_v0_think.jinja2}"
OUT_ROOT="/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval"
THINK_TEMPLATE="${THINK_TEMPLATE:-${_OTA_REPO}/chat_templates/delphi_v0_think.jinja2}"
OUT_ROOT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval}"
OUT="${OUT_ROOT}/${RUN_NAME}"
TP_SIZE=2
MAX_MODEL_LEN=4096
Expand Down
40 changes: 32 additions & 8 deletions eval/evalchemy/evalchemy_eval.sbatch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,14 @@
#SBATCH --gres=gpu:4
#SBATCH --time=08:00:00
#SBATCH --job-name=evalchemy-eval
#SBATCH --output=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/%x-%j.log
#SBATCH --output=/leonardo_work/AIFAC_5C0_290/%u/experiments/delphi-eval/%x-%j.log
#
# PREREQUISITE (multi-operator): the --output directory above must EXIST BEFORE SUBMIT.
# SLURM opens the log file before the job script runs and does NOT create intermediate
# directories -- a missing dir fails the job instantly with "Unable to open file", which
# looks like a launch bug rather than a missing mkdir. It already exists for the original
# operator; a NEW operator runs this once:
# mkdir -p /leonardo_work/AIFAC_5C0_290/$USER/experiments/delphi-eval
#
# ==============================================================================
# evalchemy_eval.sbatch — GENERIC evalchemy downstream-eval runner (MATH500 + gsm8k + AIME24)
Expand Down Expand Up @@ -42,19 +49,36 @@

set -euo pipefail

# ----- $USER-scoped paths (multi-operator) -----------------------------------------------------
# Every path below resolves to the ORIGINAL bfeuer00 string byte-identically when USER=bfeuer00,
# so this is a no-op for the original operator. Read-only shared runtime (conda env, the canonical
# evalchemy-marin clone, nltk data) falls back to the bfeuer00 copy when the submitter has none —
# a second operator does NOT need to duplicate a 20 GB env to run an eval. WRITE paths
# (OUT_ROOT, CACHE_ROOT, HF_HOME) are always per-user: bfeuer00's dirs are mode 755.
_OTA_WORK="/leonardo_work/AIFAC_5C0_290/${USER:-$(id -un)}"
_OTA_SHARED="/leonardo_work/AIFAC_5C0_290/bfeuer00"
_pick() { [ -e "$1" ] && echo "$1" || echo "$2"; }
# Conda: pick the root that actually CONTAINS the env. Testing only for a miniforge3 dir is not
# enough -- an operator can have a partial install (e.g. otagent only) and would then fail with
# "EnvironmentNameNotFound: evalchemy-marin" instead of falling back to the shared root.
_pick_conda() { [ -d "${_OTA_WORK}/miniforge3/envs/$1" ] && echo "${_OTA_WORK}/miniforge3" || echo "${_OTA_SHARED}/miniforge3"; } # own copy if present, else shared

# ----- env: conda env `evalchemy-marin` (lm-eval; the env every recent SCORES.md row used) -----
source /leonardo_work/AIFAC_5C0_290/bfeuer00/miniforge3/etc/profile.d/conda.sh
_OTA_CONDA_ROOT="${OTAGENT_CONDA_ROOT:-$(_pick_conda evalchemy-marin)}"
source "${_OTA_CONDA_ROOT}/etc/profile.d/conda.sh"
conda activate evalchemy-marin
# Must cd into the evalchemy-MARIN clone (matches the env's editable install). evalchemy-marin's
# eval/eval.py imports parse_eval_args/setup_parser from its eval/lm_eval_compat.py shim — unlike
# the now-removed `code/evalchemy` clone, whose eval/eval.py did `from lm_eval.__main__ import
# parse_eval_args, setup_parser` (both dropped in lm-eval 0.4.12 -> ImportError at startup).
# (`code/evalchemy` + its `evalchemy-resume-test` worktree were deleted 2026-06-18; this is the sole clone.)
cd /leonardo_work/AIFAC_5C0_290/bfeuer00/code/evalchemy-marin
cd "${EVALCHEMY_DIR:-$(_pick "${_OTA_WORK}/code/evalchemy-marin" "${_OTA_SHARED}/code/evalchemy-marin")}"

source ~/secrets.env # HF_TOKEN etc. (BEFORE HOME override)
export HF_HOME=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub
export HF_HUB_CACHE=/leonardo_work/AIFAC_5C0_290/bfeuer00/data/hub
# HF cache is per-user and WRITABLE (model weights may be symlinked in from another operator's cache;
# HF's python resolver follows symlinks — datasets must be real-copied, see ops/leonardo).
export HF_HOME="${HF_HOME:-${_OTA_WORK}/data/hub}"
export HF_HUB_CACHE="${HF_HUB_CACHE:-${_OTA_WORK}/data/hub}"
export HF_HUB_OFFLINE=1 # repos pre-cached on login node; no compute-node internet
export HF_ALLOW_CODE_EVAL=1 # required for gsm8k execution grading
export TOKENIZERS_PARALLELISM=false
Expand All @@ -64,12 +88,12 @@ export HF_HUB_ENABLE_HF_TRANSFER=1 # only matters for any pre-download; harmles
# point NLTK_DATA at the pre-downloaded copy under $WORK (populated once on a login node:
# python -c 'import nltk; d="$WORK/data/nltk_data"; nltk.download("punkt_tab",download_dir=d); nltk.download("punkt",download_dir=d)'
# ). Harmless for the math tasks (they don't import nltk).
export NLTK_DATA="${NLTK_DATA:-/leonardo_work/AIFAC_5C0_290/bfeuer00/data/nltk_data}"
export NLTK_DATA="${NLTK_DATA:-$(_pick "${_OTA_WORK}/data/nltk_data" "${_OTA_SHARED}/data/nltk_data")}"

# ----- writable caches (Leonardo: $HOME is READ-ONLY from compute AND login nodes) -----
# vLLM's flashinfer/triton/inductor JIT caches default under $HOME/.cache and hard-crash every worker
# on a read-only HOME. Redirect HOME + every cache to the writable work filesystem.
CACHE_ROOT=/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval/.cache
CACHE_ROOT="${CACHE_ROOT:-${_OTA_WORK}/experiments/delphi-eval/.cache}" # must be WRITABLE
mkdir -p "$CACHE_ROOT"/{flashinfer,triton,inductor,vllm,xdg,hf_local}
export HOME="$CACHE_ROOT/hf_local" # flashinfer derives FLASHINFER_WORKSPACE_DIR from $HOME/.cache
export XDG_CACHE_HOME="$CACHE_ROOT/xdg"
Expand All @@ -83,7 +107,7 @@ mkdir -p "$HOME/.cache/flashinfer"
MODEL_REPO="${1:?usage: sbatch --job-name=<PREFIX>-<RUN_NAME> evalchemy_eval.sbatch <MODEL_REPO> <RUN_NAME> [STAGE]}"
RUN_NAME="${2:?usage: sbatch --job-name=<PREFIX>-<RUN_NAME> evalchemy_eval.sbatch <MODEL_REPO> <RUN_NAME> [STAGE]}"
STAGE="${3:-sft}"
OUT_ROOT="${OUT_ROOT:-/leonardo_work/AIFAC_5C0_290/bfeuer00/experiments/delphi-eval}"
OUT_ROOT="${OUT_ROOT:-${_OTA_WORK}/experiments/delphi-eval}"
JOB_PREFIX="${JOB_PREFIX:-evalchemy-eval}"
MAX_GEN_TOKS="${MAX_GEN_TOKS:-30720}"
GPUS_PER_NODE="${GPUS_PER_NODE:-4}"
Expand Down
Loading