From 9fda90b350372160469e5f6b12f375580fa01065 Mon Sep 17 00:00:00 2001 From: pasha Date: Thu, 6 Aug 2026 15:51:28 +0300 Subject: [PATCH] fix(train): add retrieval replay to SFT and repair broken replay data paths Two problems with the SFT workflow, one measured and one that would have failed the run at launch. Measured: retrieval capability decays during causal-only training. Held-out top-1 on the advanced split fell 42.5% (step 249k peak) -> 34.4% -> 28.8% -> 21.3% (step 392k) across ~143k causal-only steps, with university 31.9% -> 17.2% and school 22.4% -> 12.5%; recall@10 is now falling too, so the embedding is losing documents outright rather than merely ranking them worse. Training loss cannot show this because it only reflects the objective currently running. SFT is five consecutive supervised-generation phases, so it would erode the retriever -- the actual product -- the same way. Four retrieval_representation replay phases are now interleaved after each SFT phase at layer 24 / temperature 0.05, using each pretraining stage's own retrieval shard and geometry: 8.1% of tokens, alongside the existing 15.7% causal foundation replay. The replay dose is a starting point, not a derived optimum. Verify it by running `hermes-train eval --objective contrastive_retrieval` on all three held-out splits before and after SFT and comparing against the 249k peak; if retrieval still regresses, raise the dose rather than accepting it. Broken paths: all four causal replay phases pointed at /opt/hermes-run/moe-300m-v5-optimized-20260731/data/, which does not exist on the training host. The shards live under /opt/hermes-run/moe-300m-v4/data/. Every replay phase would have failed to open its data. Retrieval phase geometry is sized against the validator's worst case of 33 sequences per example (MAX_RETRIEVAL_REPRESENTATION_DOCUMENTS), not the 4 our data actually carries, so the batch and shuffle bounds hold for any negatives count the schema permits. Workflow validates against retriever_300m_moe.mal: 13 phases, 18,100 steps, 915.9M compute tokens. --- hermes-train/workflow.sft.example.json | 96 +++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 8 deletions(-) diff --git a/hermes-train/workflow.sft.example.json b/hermes-train/workflow.sft.example.json index 0dc48d3c..300dbc20 100644 --- a/hermes-train/workflow.sft.example.json +++ b/hermes-train/workflow.sft.example.json @@ -19,11 +19,31 @@ "loss_weight": 1.0, "learning_rate_scale": 1.0 }, + { + "name": "replay-retrieval-language-foundations-seq512", + "type": "continued_pretrain", + "task": { + "type": "retrieval_representation", + "temperature": 0.05, + "layer": 24 + }, + "data": "/opt/hermes-run/moe-300m-v4/data/01-language-foundations-retrieval.jsonl.zst", + "sequence_length": 512, + "batch_size": 8, + "gradient_accumulation": 4, + "epochs": 1, + "shuffle_buffer": 4096, + "steps": 1200, + "loss_weight": 1.0, + "learning_rate_scale": 0.5 + }, { "name": "replay-1-language-foundations-seq512", "type": "continued_pretrain", - "task": { "type": "causal_lm" }, - "data": "/opt/hermes-run/moe-300m-v5-optimized-20260731/data/01-language-foundations-causal.jsonl.zst", + "task": { + "type": "causal_lm" + }, + "data": "/opt/hermes-run/moe-300m-v4/data/01-language-foundations-causal.jsonl.zst", "sequence_length": 512, "batch_size": 32, "gradient_accumulation": 2, @@ -50,11 +70,31 @@ "loss_weight": 1.0, "learning_rate_scale": 1.0 }, + { + "name": "replay-retrieval-school-fundamentals-seq512", + "type": "continued_pretrain", + "task": { + "type": "retrieval_representation", + "temperature": 0.05, + "layer": 24 + }, + "data": "/opt/hermes-run/moe-300m-v4/data/02-school-fundamentals-retrieval.jsonl.zst", + "sequence_length": 512, + "batch_size": 8, + "gradient_accumulation": 4, + "epochs": 1, + "shuffle_buffer": 4096, + "steps": 1200, + "loss_weight": 1.0, + "learning_rate_scale": 0.5 + }, { "name": "replay-2-school-fundamentals-seq1024", "type": "continued_pretrain", - "task": { "type": "causal_lm" }, - "data": "/opt/hermes-run/moe-300m-v5-optimized-20260731/data/02-school-fundamentals-causal.jsonl.zst", + "task": { + "type": "causal_lm" + }, + "data": "/opt/hermes-run/moe-300m-v4/data/02-school-fundamentals-causal.jsonl.zst", "sequence_length": 1024, "batch_size": 16, "gradient_accumulation": 2, @@ -82,6 +122,24 @@ "loss_weight": 1.0, "learning_rate_scale": 1.0 }, + { + "name": "replay-retrieval-university-core-seq768", + "type": "continued_pretrain", + "task": { + "type": "retrieval_representation", + "temperature": 0.05, + "layer": 24 + }, + "data": "/opt/hermes-run/moe-300m-v4/data/03-university-core-retrieval.jsonl.zst", + "sequence_length": 768, + "batch_size": 8, + "gradient_accumulation": 2, + "epochs": 1, + "shuffle_buffer": 4096, + "steps": 1200, + "loss_weight": 1.0, + "learning_rate_scale": 0.5 + }, { "name": "sft-4-qa-reasoning-rag-seq2048", "type": "sft", @@ -100,11 +158,31 @@ "loss_weight": 1.0, "learning_rate_scale": 1.0 }, + { + "name": "replay-retrieval-advanced-scholarship-seq1024", + "type": "continued_pretrain", + "task": { + "type": "retrieval_representation", + "temperature": 0.05, + "layer": 24 + }, + "data": "/opt/hermes-run/moe-300m-v4/data/04-advanced-scholarship-retrieval.jsonl.zst", + "sequence_length": 1024, + "batch_size": 4, + "gradient_accumulation": 4, + "epochs": 1, + "shuffle_buffer": 2048, + "steps": 1200, + "loss_weight": 1.0, + "learning_rate_scale": 0.5 + }, { "name": "replay-3-university-core-seq2048", "type": "continued_pretrain", - "task": { "type": "causal_lm" }, - "data": "/opt/hermes-run/moe-300m-v5-optimized-20260731/data/03-university-core-causal.jsonl.zst", + "task": { + "type": "causal_lm" + }, + "data": "/opt/hermes-run/moe-300m-v4/data/03-university-core-causal.jsonl.zst", "sequence_length": 2048, "batch_size": 8, "gradient_accumulation": 2, @@ -134,8 +212,10 @@ { "name": "replay-4-advanced-scholarship-seq4096", "type": "continued_pretrain", - "task": { "type": "causal_lm" }, - "data": "/opt/hermes-run/moe-300m-v5-optimized-20260731/data/04-advanced-scholarship-causal.jsonl.zst", + "task": { + "type": "causal_lm" + }, + "data": "/opt/hermes-run/moe-300m-v4/data/04-advanced-scholarship-causal.jsonl.zst", "sequence_length": 4096, "batch_size": 4, "gradient_accumulation": 2,