Eval bug: llama-server --embedding got Corrupted Data, quality decline

[fgst666]

Name and Version

build 10175 (60bccc3) with Clang 20.1.8 for Windows x86_64

Operating systems

Windows

GGML backends

CPU

Hardware

i5 10400

Models

jina-embeddings-v5-nano-retrieval-Q5_K_M.gguf

Problem description & steps to reproduce

llama-server --embedding got wrong data
Incorrect data output only happens after requests.
For the same text, the vector obtained from the first request after each cold start is always exactly the same.
The bug doesn't produce completely wrong output — it causes a noticeable quality drop, with an L2 distance of about 0.02 (for normalized vectors). (This may depend on how much the cache has been polluted. just I think)
llama-server -m G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf --embedding --pooling last -c 2048 -b 64 -ub 64 --load-mode dio --port 8082 --cache-ram 0 -lv 4 -fa off --threads 1 -fit off
llama-server defaults to 4 slots with kvu enabled. When I manually specify -np 4 (which defaults to -no-kvu), the bug does not occur. this occasion still has bug.

def verifyCooling(txt:str):
    vecA = getVec(txt)
    print("A:", vecA[:5])
    input("restart llama-server")
    vecB = getVec(txt)
    print("B:", vecB[:5])
    l2d = computeL2(vecA, vecB)
    print("all cooling L2 =", l2d)
    print()

this fine, same, L2=0 , OK

A: [-0.06865525245666504, 0.02405271865427494, -0.07321952283382416, -0.0781535804271698, 0.12095746397972107]
restart llama-server
B: [-0.06865525245666504, 0.02405271865427494, -0.07321952283382416, -0.0781535804271698, 0.12095746397972107]
all cooling L2 = 0.0

---

but after another requests, input same text, got wrong

def outCache():
    # for _ in range(5):
    for _ in range(random.randint(5, 8)):
        # t = f"{random.random():.5f}" * 20
        t = [f"{random.random():.5f}" for _ in range(random.randint(16, 24))]
        _ = getVec("  ".join(t))

def test(txt:str):
    outCache()
    vecA = getVec(txt)
    print("A:", vecA[:5])
    outCache()
    vecB = getVec(txt)
    print("B:", vecB[:5])
    l2d = computeL2(vecA, vecB)
    print(f"L2 = {l2d}\n")

If the commented-out code is used instead, the results look like this:

restart llama-server1
B: [-0.06758762151002884, 0.0248175747692585, -0.07288508117198944, -0.07876278460025787, 0.12040942907333374]
all cooling L2 = 0.0

A: [-0.06671811640262604, 0.025068677961826324, -0.07316775619983673, -0.07915324717760086, 0.12086104601621628]
B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375]
L2 = 0.025976544239568217

A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385]
B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375]
L2 = 0.025698335751558988

A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385]
B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375]
L2 = 0.025698335751558988

A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385]
B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375]
L2 = 0.025698335751558988

A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385]
B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375]
L2 = 0.025698335751558988

---

all code

import json, math, random

import requests


def computeL2(a:list[float], b:list[float]):
    return math.sqrt(sum((x-y)**2 for x,y in zip(a,b)))


def getVec(t:str) -> list[float]:
    body = {
        "input": t,
        "encoding_format": "float"
    }
    with requests.post("http://127.0.0.1:8082/v1/embeddings", json=body) as resp:
        resp.raise_for_status()
        rbody = json.loads(resp.content)
        return rbody["data"][0]["embedding"]


def outCache():
    # for _ in range(5):
    for _ in range(random.randint(5, 8)):
        # t = f"{random.random():.5f}" * 20
        t = [f"{random.random():.5f}" for _ in range(random.randint(16, 24))]
        _ = getVec("  ".join(t))


def test(txt:str):
    outCache()
    vecA = getVec(txt)
    print("A:", vecA[:5])
    outCache()
    vecB = getVec(txt)
    print("B:", vecB[:5])
    l2d = computeL2(vecA, vecB)
    print(f"L2 = {l2d}\n")


def verifyCooling(txt:str):
    vecA = getVec(txt)
    print("A:", vecA[:5])
    input("restart llama-server")
    vecB = getVec(txt)
    print("B:", vecB[:5])
    l2d = computeL2(vecA, vecB)
    print("all cooling L2 =", l2d)
    print()


TEXT = """b10174
@github-actions github-actions released this 1 hour ago
    b10174
    7be2c65 
model: add NextN/MTP speculative decoding support for GLM_DSA (GLM-5.2) (#25980)

model: add NextN/MTP speculative decoding support for GLM_DSA (GLM-5.2)
Adds GLM-5.2 NextN/MTP as a --spec-type draft-mtp target: nextn tensor
loading via the qwen35moe/step35-style presence probe, a graph_mtp
builder (enorm/hnorm/eh_proj + dense MLA + sigmoid-gated MoE with
shared expert + shared head with fallbacks, _s scale tensors passed
for NVFP4), t_h_nextn extraction in the trunk graph, and MTP-context
KV setup: the draft head runs dense MLA, so the MTP context uses a
plain attention KV cache holding only the nextn layer(s) (same
pattern as the hybrid Qwen3.5 MTP context) while the main context
keeps the DSA cache, now filtered to trunk layers only.

Co-Authored-By: Claude Fable 5 noreply@anthropic.com

convert : support --mtp/--no-mtp export for GlmMoeDsaForCausalLM (GLM-5.2)
Opt GLM-5.2 into the supports_mtp_export contract (post-#25641 shape,
mirroring HYV3Model/Step35Model): --no-mtp drops the appended NextN
block (blk.78) and its nextn_predict_layers KV; --mtp keeps only the
NextN block plus shared embeddings/norm/lm_head. Default (bundled)
output is unchanged.

Co-Authored-By: Claude Fable 5 noreply@anthropic.com

Co-authored-by: Claude Fable 5 noreply@anthropic.com

Website:

https://llama.app
macOS/iOS:

macOS Apple Silicon (arm64)
macOS Apple Silicon (arm64, KleidiAI enabled) DISABLED
macOS Intel (x64)
iOS XCFramework
Linux:

Ubuntu x64 (CPU)
Ubuntu arm64 (CPU)"""

if __name__ == "__main__":
    verifyCooling(TEXT)
    for i in range(5):
        # break
        test(TEXT)

First Bad Commit

No response

Relevant log output

Logs

0.00.032.796 I cmn  common_param: common_params_print_info: build 10175 (60bccc376) with Clang 20.1.8 for Windows x86_640.00.032.799 I cmn  common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg)
0.00.032.801 I cmn  common_param: device_info:
0.00.032.805 I cmn  common_param:   - CPU     : Intel(R) Core(TM) i5-10400 CPU @ 2.90GHz (8018 MiB, 2663 MiB free)
0.00.032.847 I cmn  common_param: system_info: n_threads = 1 (n_threads_batch = 1) / 12 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
0.00.032.851 I srv  llama_server: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true
0.00.032.891 I srv          init: running without SSL
0.00.032.975 I srv          init: using 11 threads for HTTP server
0.00.033.267 W srv  llama_server: -----------------
0.00.033.268 W srv  llama_server: CORS is set to allow all origins ('*') and no API key is set
0.00.033.269 W srv  llama_server: this can be a security risk (cross-origin attacks)
0.00.033.270 W srv  llama_server: more info: https://github.com/ggml-org/llama.cpp/pull/25655
0.00.033.270 W srv  llama_server: -----------------
0.00.033.296 I srv         start: binding port with default address family
0.00.042.004 I srv    load_model: loading model 'G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf'
0.00.042.008 I srv    load_model: local path 'G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf'
0.00.101.464 I llama_model_loader: loaded meta data with 38 key-value pairs and 310 tensors from G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf (version GGUF V3 (latest))
0.00.101.480 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
0.00.101.488 I llama_model_loader: - kv   0:                       general.architecture str              = qwen3
0.00.101.489 I llama_model_loader: - kv   1:                               general.type str              = model
0.00.101.493 I llama_model_loader: - kv   2:                               general.name str              = Jina Embeddings v5 Text Small Retrieval
0.00.101.495 I llama_model_loader: - kv   3:                           general.finetune str              = retrieval
0.00.101.496 I llama_model_loader: - kv   4:                           general.basename str              = jina-embeddings-v5-text
0.00.101.497 I llama_model_loader: - kv   5:                         general.size_label str              = small
0.00.101.498 I llama_model_loader: - kv   6:                            general.license str              = cc
0.00.101.501 I llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
0.00.101.502 I llama_model_loader: - kv   8:                  general.base_model.0.name str              = Qwen3 0.6B
0.00.101.503 I llama_model_loader: - kv   9:          general.base_model.0.organization str              = Qwen
0.00.101.505 I llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/Qwen/Qwen3-0.6B
0.00.101.507 I llama_model_loader: - kv  11:                          qwen3.block_count u32              = 28
0.00.101.508 I llama_model_loader: - kv  12:                       qwen3.context_length u32              = 40960
0.00.101.509 I llama_model_loader: - kv  13:                     qwen3.embedding_length u32              = 1024
0.00.101.509 I llama_model_loader: - kv  14:                  qwen3.feed_forward_length u32              = 3072
0.00.101.510 I llama_model_loader: - kv  15:                 qwen3.attention.head_count u32              = 16
0.00.101.511 I llama_model_loader: - kv  16:              qwen3.attention.head_count_kv u32              = 8
0.00.101.517 I llama_model_loader: - kv  17:                       qwen3.rope.freq_base f32              = 3500000.000000
0.00.101.519 I llama_model_loader: - kv  18:     qwen3.attention.layer_norm_rms_epsilon f32              = 0.000001
0.00.101.520 I llama_model_loader: - kv  19:                 qwen3.attention.key_length u32              = 128
0.00.101.521 I llama_model_loader: - kv  20:               qwen3.attention.value_length u32              = 128
0.00.101.522 I llama_model_loader: - kv  21:                       tokenizer.ggml.model str              = gpt2
0.00.101.522 I llama_model_loader: - kv  22:                         tokenizer.ggml.pre str              = qwen2
0.00.139.665 I llama_model_loader: - kv  23:                      tokenizer.ggml.tokens arr[str,151936]  = ["!", "\"", "#", "$", "%", "&", "'", ...
0.00.151.009 I llama_model_loader: - kv  24:                  tokenizer.ggml.token_type arr[i32,151936]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
0.00.189.852 I llama_model_loader: - kv  25:                      tokenizer.ggml.merges arr[str,151387]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
0.00.189.857 I llama_model_loader: - kv  26:                tokenizer.ggml.eos_token_id u32              = 151645
0.00.189.858 I llama_model_loader: - kv  27:            tokenizer.ggml.padding_token_id u32              = 151643
0.00.189.859 I llama_model_loader: - kv  28:                tokenizer.ggml.bos_token_id u32              = 151643
0.00.189.860 I llama_model_loader: - kv  29:               tokenizer.ggml.add_bos_token bool             = false
0.00.189.866 I llama_model_loader: - kv  30:                    tokenizer.chat_template str              = {%- if tools %}\n    {{- '<|im_start|>...
0.00.189.867 I llama_model_loader: - kv  31:               general.quantization_version u32              = 2
0.00.189.868 I llama_model_loader: - kv  32:                          general.file_type u32              = 15
0.00.189.870 I llama_model_loader: - kv  33:                      quantize.imatrix.file str              = /home/mguenther/multimodal-large-scal...
0.00.189.871 I llama_model_loader: - kv  34:                   quantize.imatrix.dataset str              = data/calibration_data_v5_rc.txt
0.00.189.872 I llama_model_loader: - kv  35:             quantize.imatrix.entries_count u32              = 196
0.00.189.873 I llama_model_loader: - kv  36:              quantize.imatrix.chunks_count u32              = 225
0.00.189.874 I llama_model_loader: - kv  37:                         qwen3.pooling_type u32              = 3
0.00.189.875 I llama_model_loader: - type  f32:  113 tensors
0.00.189.875 I llama_model_loader: - type q4_K:  168 tensors
0.00.189.876 I llama_model_loader: - type q6_K:   29 tensors
0.00.189.881 I print_info: file format = GGUF V3 (latest)
0.00.189.881 I print_info: file type   = Q4_K - Medium
0.00.189.885 I print_info: file size   = 372.65 MiB (5.24 BPW)
0.00.302.760 I load: 0 unused tokens
0.00.322.104 W load: control-looking token: 128247 '</s>' was not control-type; this is probably a bug in the model. its type will be overridden
0.00.327.844 I load: printing all EOG tokens:
0.00.327.847 I load:   - 128247 ('</s>')
0.00.327.847 I load:   - 151643 ('<|endoftext|>')
0.00.327.848 I load:   - 151645 ('<|im_end|>')
0.00.327.848 I load:   - 151662 ('<|fim_pad|>')
0.00.327.848 I load:   - 151663 ('<|repo_name|>')
0.00.327.849 I load:   - 151664 ('<|file_sep|>')
0.00.328.339 I load: special tokens cache size = 27
0.00.368.578 I load: token to piece cache size = 0.9311 MB
0.00.368.606 I print_info: arch                  = qwen3
0.00.368.608 I print_info: vocab_only            = 0
0.00.368.608 I print_info: no_alloc              = 0
0.00.368.608 I print_info: n_ctx_train           = 40960
0.00.368.609 I print_info: n_embd_inp            = 1024
0.00.368.609 I print_info: n_embd                = 1024
0.00.368.610 I print_info: n_embd_out            = 1024
0.00.368.610 I print_info: n_layer               = 28
0.00.368.610 I print_info: n_layer_all           = 28
0.00.368.638 I print_info: n_head                = 16
0.00.368.646 I print_info: n_head_kv             = 8
0.00.368.647 I print_info: n_rot                 = 128
0.00.368.647 I print_info: n_swa                 = 0
0.00.368.647 I print_info: is_swa_any            = 0
0.00.368.648 I print_info: n_embd_head_k         = 128
0.00.368.648 I print_info: n_embd_head_v         = 128
0.00.368.655 I print_info: n_gqa                 = 2
0.00.368.662 I print_info: n_embd_k_gqa          = 1024
0.00.368.678 I print_info: n_embd_v_gqa          = 1024
0.00.368.683 I print_info: f_norm_eps            = 0.0e+00
0.00.368.684 I print_info: f_norm_rms_eps        = 1.0e-06
0.00.368.684 I print_info: f_clamp_kqv           = 0.0e+00
0.00.368.685 I print_info: f_max_alibi_bias      = 0.0e+00
0.00.368.685 I print_info: f_logit_scale         = 0.0e+00
0.00.368.686 I print_info: f_attn_scale          = 0.0e+00
0.00.368.686 I print_info: f_attn_value_scale    = 0.0000
0.00.368.693 I print_info: n_ff                  = 3072
0.00.368.694 I print_info: n_expert              = 0
0.00.368.694 I print_info: n_expert_used         = 0
0.00.368.695 I print_info: n_expert_groups       = 0
0.00.368.695 I print_info: n_group_used          = 0
0.00.368.695 I print_info: causal attn           = 1
0.00.368.696 I print_info: pooling type          = 3
0.00.368.696 I print_info: rope type             = 2
0.00.368.696 I print_info: rope scaling          = linear
0.00.368.698 I print_info: freq_base_train       = 3500000.0
0.00.368.699 I print_info: freq_scale_train      = 1
0.00.368.699 I print_info: n_ctx_orig_yarn       = 40960
0.00.368.700 I print_info: rope_yarn_log_mul     = 0.0000
0.00.368.700 I print_info: rope_finetuned        = unknown
0.00.368.701 I print_info: model type            = 0.6B
0.00.368.703 I print_info: model params          = 596.05 M
0.00.368.703 I print_info: general.name          = Jina Embeddings v5 Text Small Retrieval
0.00.368.704 I print_info: vocab type            = BPE
0.00.368.704 I print_info: n_vocab               = 151936
0.00.368.705 I print_info: n_merges              = 151387
0.00.368.705 I print_info: BOS token             = 151643 '<|endoftext|>'
0.00.368.706 I print_info: EOS token             = 151645 '<|im_end|>'
0.00.368.706 I print_info: EOT token             = 151645 '<|im_end|>'
0.00.368.707 I print_info: PAD token             = 151643 '<|endoftext|>'
0.00.368.707 I print_info: LF token              = 198 'Ċ'
0.00.368.708 I print_info: FIM PRE token         = 151659 '<|fim_prefix|>'
0.00.368.708 I print_info: FIM SUF token         = 151661 '<|fim_suffix|>'
0.00.368.709 I print_info: FIM MID token         = 151660 '<|fim_middle|>'
0.00.368.709 I print_info: FIM PAD token         = 151662 '<|fim_pad|>'
0.00.368.709 I print_info: FIM REP token         = 151663 '<|repo_name|>'
0.00.368.710 I print_info: FIM SEP token         = 151664 '<|file_sep|>'
0.00.368.710 I print_info: EOG token             = 128247 '</s>'
0.00.368.711 I print_info: EOG token             = 151643 '<|endoftext|>'
0.00.368.711 I print_info: EOG token             = 151645 '<|im_end|>'
0.00.368.712 I print_info: EOG token             = 151662 '<|fim_pad|>'
0.00.368.712 I print_info: EOG token             = 151663 '<|repo_name|>'
0.00.368.713 I print_info: EOG token             = 151664 '<|file_sep|>'
0.00.368.713 I print_info: max token length      = 256
0.00.368.715 I load_tensors: loading model tensors, this can take a while... (load_mode = dio)
0.00.382.521 I load_tensors: offloading output layer to GPU
0.00.382.522 I load_tensors: offloading 27 repeating layers to GPU
0.00.382.522 I load_tensors: offloaded 29/29 layers to GPU
0.00.382.526 I load_tensors:          CPU model buffer size =   167.90 MiB
0.00.382.527 I load_tensors:   CPU_REPACK model buffer size =   204.75 MiB
0.00.606.909 I cmn  common_init_: added </s> logit bias = -inf
0.00.607.078 I cmn  common_init_: added <|endoftext|> logit bias = -inf
0.00.607.087 I cmn  common_init_: added <|im_end|> logit bias = -inf
0.00.607.088 I cmn  common_init_: added <|fim_pad|> logit bias = -inf
0.00.607.089 I cmn  common_init_: added <|repo_name|> logit bias = -inf
0.00.607.089 I cmn  common_init_: added <|file_sep|> logit bias = -inf
0.00.607.763 I llama_context: constructing llama_context
0.00.607.769 I llama_context: n_seq_max     = 4
0.00.607.769 I llama_context: n_ctx         = 2048
0.00.607.770 I llama_context: n_ctx_seq     = 2048
0.00.607.771 I llama_context: n_batch       = 64
0.00.607.771 I llama_context: n_ubatch      = 64
0.00.607.771 I llama_context: causal_attn   = 1
0.00.607.772 I llama_context: flash_attn    = disabled
0.00.607.772 I llama_context: kv_unified    = true
0.00.607.776 I llama_context: freq_base     = 3500000.0
0.00.607.777 I llama_context: freq_scale    = 1
0.00.607.777 I llama_context: n_rs_seq      = 0
0.00.607.777 I llama_context: n_outputs_max = 64
0.00.607.778 I llama_context: n_ctx_seq (2048) < n_ctx_train (40960) -- the full capacity of the model will not be utilized
0.00.608.159 I llama_context:        CPU  output buffer size =     2.33 MiB
0.00.608.318 I llama_kv_cache:        CPU KV buffer size =   224.00 MiB
0.00.646.316 I llama_kv_cache: size =  224.00 MiB (  2048 cells,  28 layers,  4/1 seqs), K (f16):  112.00 MiB, V (f16):  112.00 MiB
0.00.646.330 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 128
0.00.646.330 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 128
0.00.646.348 I sched_reserve: reserving ...
0.00.647.727 I resolve_fused_ops: resolving fused Gated Delta Net support:
0.00.648.316 I resolve_fused_ops: fused Gated Delta Net (autoregressive) enabled
0.00.648.804 I resolve_fused_ops: fused Gated Delta Net (chunked) enabled
0.00.648.805 I resolve_fused_ops: resolving fused Lightning Indexer support:
0.00.649.274 I resolve_fused_ops: Lightning Indexer enabled
0.00.649.274 I resolve_fused_ops: resolving fused DeepSeek V4 HC support:
0.00.649.713 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled
0.00.650.144 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled
0.00.650.570 I resolve_fused_ops: fused DeepSeek V4 HC post enabled
0.00.652.833 I sched_reserve:        CPU compute buffer size =    38.35 MiB
0.00.652.838 I sched_reserve: graph nodes  = 1127
0.00.652.839 I sched_reserve: graph splits = 1
0.00.652.841 I sched_reserve: reserve took 6.49 ms, sched copies = 1
0.00.653.396 I cmn  common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable)
0.00.827.519 I srv    load_model: initializing, n_slots = 4, n_ctx_slot = 2048, kv_unified = 'true'
0.00.827.555 I spec common_specu: no implementations specified for speculative decoding
0.00.827.558 I slot   load_model: id  0 | task -1 | new slot, n_ctx = 2048
0.00.827.562 I slot   load_model: id  1 | task -1 | new slot, n_ctx = 2048
0.00.827.562 I slot   load_model: id  2 | task -1 | new slot, n_ctx = 2048
0.00.827.563 I slot   load_model: id  3 | task -1 | new slot, n_ctx = 2048
0.00.827.581 I srv    load_model: prompt cache is disabled - use `--cache-ram N` to enable it
0.00.827.582 I srv    load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391
0.00.827.585 I srv    load_model: context checkpoints enabled, max = 32, min spacing = 8192
0.00.827.629 W srv          init: --cache-idle-slots requires --cache-ram, disabling
0.00.847.574 I srv          init: init: chat template, example_format: '<|im_start|>system
You are a helpful assistant<|im_end|>
<|im_start|>user
Hello<|im_end|>
<|im_start|>assistant
Hi there<|im_end|>
<|im_start|>user
How are you?<|im_end|>
<|im_start|>assistant
'
0.00.858.907 I srv          init: init: chat template, thinking = 1
0.00.858.936 I srv  llama_server: model loaded
0.00.858.938 I srv  llama_server: listening on http://0.0.0.0:8082
0.00.858.945 I srv  update_slots: all slots are idle
0.02.702.599 I slot get_availabl: id  3 | task -1 | selected slot by LRU, t_last = -1
0.02.702.611 I slot launch_slot_: id  3 | task 0 | processing task, is_child = 0
0.02.702.624 I slot   operator(): id  3 | task 0 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 502
0.02.702.630 I slot   operator(): id  3 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end)
0.03.883.046 I slot   operator(): id  3 | task 0 | cached n_tokens = 64, memory_seq_rm [64, end)
0.05.054.032 I slot   operator(): id  3 | task 0 | cached n_tokens = 128, memory_seq_rm [128, end)
0.06.218.128 I slot print_timing: id  3 | task 0 | prompt processing, n_tokens =    192, progress = 0.38, t =   3.52 s / 54.62 tokens per second
0.06.218.132 I slot   operator(): id  3 | task 0 | cached n_tokens = 192, memory_seq_rm [192, end)
0.07.385.030 I slot print_timing: id  3 | task 0 | prompt processing, n_tokens =    256, progress = 0.51, t =   4.68 s / 54.67 tokens per second
0.07.385.034 I slot   operator(): id  3 | task 0 | cached n_tokens = 256, memory_seq_rm [256, end)
0.08.631.066 I slot print_timing: id  3 | task 0 | prompt processing, n_tokens =    320, progress = 0.64, t =   5.93 s / 53.98 tokens per second
0.08.631.071 I slot   operator(): id  3 | task 0 | cached n_tokens = 320, memory_seq_rm [320, end)
0.09.872.124 I slot print_timing: id  3 | task 0 | prompt processing, n_tokens =    384, progress = 0.76, t =   7.17 s / 53.56 tokens per second
0.09.872.128 I slot   operator(): id  3 | task 0 | cached n_tokens = 384, memory_seq_rm [384, end)
0.11.107.041 I slot print_timing: id  3 | task 0 | prompt processing, n_tokens =    448, progress = 0.89, t =   8.40 s / 53.31 tokens per second
0.11.107.045 I slot   operator(): id  3 | task 0 | cached n_tokens = 448, memory_seq_rm [448, end)
0.12.166.493 I slot      release: id  3 | task 0 | stop processing: n_tokens = 502, truncated = 0
0.12.166.503 I srv  update_slots: all slots are idle
0.12.169.820 I slot get_availabl: id  2 | task -1 | selected slot by LRU, t_last = -1
0.12.169.826 I slot launch_slot_: id  2 | task 9 | processing task, is_child = 0
0.12.169.832 I slot   operator(): id  2 | task 9 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398
0.12.169.836 I slot   operator(): id  2 | task 9 | cached n_tokens = 0, memory_seq_rm [0, end)
0.13.505.340 I slot   operator(): id  2 | task 9 | cached n_tokens = 64, memory_seq_rm [64, end)
0.14.820.498 I slot   operator(): id  2 | task 9 | cached n_tokens = 128, memory_seq_rm [128, end)
0.16.143.926 I slot print_timing: id  2 | task 9 | prompt processing, n_tokens =    192, progress = 0.48, t =   3.97 s / 48.31 tokens per second
0.16.143.931 I slot   operator(): id  2 | task 9 | cached n_tokens = 192, memory_seq_rm [192, end)
0.17.459.193 I slot print_timing: id  2 | task 9 | prompt processing, n_tokens =    256, progress = 0.64, t =   5.29 s / 48.40 tokens per second
0.17.459.197 I slot   operator(): id  2 | task 9 | cached n_tokens = 256, memory_seq_rm [256, end)
0.18.856.102 I slot print_timing: id  2 | task 9 | prompt processing, n_tokens =    320, progress = 0.80, t =   6.69 s / 47.86 tokens per second
0.18.856.106 I slot   operator(): id  2 | task 9 | cached n_tokens = 320, memory_seq_rm [320, end)
0.20.280.738 I slot print_timing: id  2 | task 9 | prompt processing, n_tokens =    384, progress = 0.96, t =   8.11 s / 47.34 tokens per second
0.20.280.743 I slot   operator(): id  2 | task 9 | cached n_tokens = 384, memory_seq_rm [384, end)
0.20.671.692 I slot      release: id  2 | task 9 | stop processing: n_tokens = 398, truncated = 0
0.20.671.740 I srv  update_slots: all slots are idle
0.20.677.081 I slot get_availabl: id  1 | task -1 | selected slot by LRU, t_last = -1
0.20.677.088 I slot launch_slot_: id  1 | task 17 | processing task, is_child = 0
0.20.677.096 I slot   operator(): id  1 | task 17 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398
0.20.677.099 I slot   operator(): id  1 | task 17 | cached n_tokens = 0, memory_seq_rm [0, end)
0.22.121.895 I slot   operator(): id  1 | task 17 | cached n_tokens = 64, memory_seq_rm [64, end)
0.23.653.748 I slot   operator(): id  1 | task 17 | cached n_tokens = 128, memory_seq_rm [128, end)
0.25.195.515 I slot print_timing: id  1 | task 17 | prompt processing, n_tokens =    192, progress = 0.48, t =   4.52 s / 42.49 tokens per second
0.25.195.519 I slot   operator(): id  1 | task 17 | cached n_tokens = 192, memory_seq_rm [192, end)
0.26.750.740 I slot print_timing: id  1 | task 17 | prompt processing, n_tokens =    256, progress = 0.64, t =   6.07 s / 42.15 tokens per second
0.26.750.744 I slot   operator(): id  1 | task 17 | cached n_tokens = 256, memory_seq_rm [256, end)
0.28.267.570 I slot print_timing: id  1 | task 17 | prompt processing, n_tokens =    320, progress = 0.80, t =   7.59 s / 42.16 tokens per second
0.28.267.574 I slot   operator(): id  1 | task 17 | cached n_tokens = 320, memory_seq_rm [320, end)
0.29.939.111 I slot print_timing: id  1 | task 17 | prompt processing, n_tokens =    384, progress = 0.96, t =   9.26 s / 41.46 tokens per second
0.29.939.116 I slot   operator(): id  1 | task 17 | cached n_tokens = 384, memory_seq_rm [384, end)
0.30.358.747 I slot      release: id  1 | task 17 | stop processing: n_tokens = 398, truncated = 0
0.30.358.758 I srv  update_slots: all slots are idle
0.30.361.726 I slot get_availabl: id  0 | task -1 | selected slot by LRU, t_last = -1
0.30.361.734 I slot launch_slot_: id  0 | task 25 | processing task, is_child = 0
0.30.361.741 I slot   operator(): id  0 | task 25 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398
0.30.361.744 I slot   operator(): id  0 | task 25 | cached n_tokens = 0, memory_seq_rm [0, end)
0.31.975.433 I slot   operator(): id  0 | task 25 | cached n_tokens = 64, memory_seq_rm [64, end)
0.33.600.319 I slot print_timing: id  0 | task 25 | prompt processing, n_tokens =    128, progress = 0.32, t =   3.24 s / 39.52 tokens per second
0.33.600.324 I slot   operator(): id  0 | task 25 | cached n_tokens = 128, memory_seq_rm [128, end)
0.35.227.110 I slot print_timing: id  0 | task 25 | prompt processing, n_tokens =    192, progress = 0.48, t =   4.87 s / 39.46 tokens per second
0.35.227.114 I slot   operator(): id  0 | task 25 | cached n_tokens = 192, memory_seq_rm [192, end)
0.36.945.258 I slot print_timing: id  0 | task 25 | prompt processing, n_tokens =    256, progress = 0.64, t =   6.58 s / 38.89 tokens per second
0.36.945.262 I slot   operator(): id  0 | task 25 | cached n_tokens = 256, memory_seq_rm [256, end)
0.38.611.448 I slot print_timing: id  0 | task 25 | prompt processing, n_tokens =    320, progress = 0.80, t =   8.25 s / 38.79 tokens per second
0.38.611.453 I slot   operator(): id  0 | task 25 | cached n_tokens = 320, memory_seq_rm [320, end)
0.40.345.882 I slot print_timing: id  0 | task 25 | prompt processing, n_tokens =    384, progress = 0.96, t =   9.98 s / 38.46 tokens per second
0.40.345.887 I slot   operator(): id  0 | task 25 | cached n_tokens = 384, memory_seq_rm [384, end)
0.40.781.556 I slot      release: id  0 | task 25 | stop processing: n_tokens = 398, truncated = 0
0.40.781.566 I srv  update_slots: all slots are idle

[zzffesgg58]

Thanks for the clarification. Understood. I'll just open a PR if I have a fix rather than asking for assignment. If someone beats me to it, that's perfectly fine.

…

On Wed, Jul 29, 2026 at 3:01 PM fgst666 ***@***.***> wrote: *fgst666* created an issue (ggml-org/llama.cpp#26282) <#26282> Name and Version build 10175 (60bccc3 <60bccc3>) with Clang 20.1.8 for Windows x86_64 Operating systems Windows GGML backends CPU Hardware i5 10400 Models jina-embeddings-v5-nano-retrieval-Q5_K_M.gguf Problem description & steps to reproduce llama-server --embedding got wrong data Incorrect data output only happens after requests. For the same text, the vector obtained from the first request after each cold start is always exactly the same. The bug doesn't produce completely wrong output — it causes a noticeable quality drop, with an L2 distance of about 0.02 (for normalized vectors). (This may depend on how much the cache has been polluted. just I think) llama-server -m G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf --embedding --pooling last -c 2048 -b 64 -ub 64 --load-mode dio --port 8082 --cache-ram 0 -lv 4 -fa off --threads 1 -fit off llama-server defaults to 4 slots with kvu enabled. When I manually specify -np 4 (which defaults to -no-kvu), the bug does not occur. def verifyCooling(txt:str): vecA = getVec(txt) print("A:", vecA[:5]) input("restart llama-server") vecB = getVec(txt) print("B:", vecB[:5]) l2d = computeL2(vecA, vecB) print("all cooling L2 =", l2d) print() this fine, same, L2=0 , OK A: [-0.06865525245666504, 0.02405271865427494, -0.07321952283382416, -0.0781535804271698, 0.12095746397972107] restart llama-server B: [-0.06865525245666504, 0.02405271865427494, -0.07321952283382416, -0.0781535804271698, 0.12095746397972107] all cooling L2 = 0.0 ------------------------------ but after another requests, input same text, got wrong def outCache(): # for _ in range(5): for _ in range(random.randint(5, 8)): # t = f"{random.random():.5f}" * 20 t = [f"{random.random():.5f}" for _ in range(random.randint(16, 24))] _ = getVec(" ".join(t)) def test(txt:str): outCache() vecA = getVec(txt) print("A:", vecA[:5]) outCache() vecB = getVec(txt) print("B:", vecB[:5]) l2d = computeL2(vecA, vecB) print(f"L2 = {l2d}\n") If the commented-out code is used instead, the results look like this: restart llama-server1 B: [-0.06758762151002884, 0.0248175747692585, -0.07288508117198944, -0.07876278460025787, 0.12040942907333374] all cooling L2 = 0.0 A: [-0.06671811640262604, 0.025068677961826324, -0.07316775619983673, -0.07915324717760086, 0.12086104601621628] B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] L2 = 0.025976544239568217 A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] L2 = 0.025698335751558988 A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] L2 = 0.025698335751558988 A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] L2 = 0.025698335751558988 A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] L2 = 0.025698335751558988 ------------------------------ all code import json, math, random import requests def computeL2(a:list[float], b:list[float]): return math.sqrt(sum((x-y)**2 for x,y in zip(a,b))) def getVec(t:str) -> list[float]: body = { "input": t, "encoding_format": "float" } with requests.post("http://127.0.0.1:8082/v1/embeddings", json=body) as resp: resp.raise_for_status() rbody = json.loads(resp.content) return rbody["data"][0]["embedding"] def outCache(): # for _ in range(5): for _ in range(random.randint(5, 8)): # t = f"{random.random():.5f}" * 20 t = [f"{random.random():.5f}" for _ in range(random.randint(16, 24))] _ = getVec(" ".join(t)) def test(txt:str): outCache() vecA = getVec(txt) print("A:", vecA[:5]) outCache() vecB = getVec(txt) print("B:", vecB[:5]) l2d = computeL2(vecA, vecB) print(f"L2 = {l2d}\n") def verifyCooling(txt:str): vecA = getVec(txt) print("A:", vecA[:5]) input("restart llama-server") vecB = getVec(txt) print("B:", vecB[:5]) l2d = computeL2(vecA, vecB) print("all cooling L2 =", l2d) print() TEXT = ***@***.*** github-actions released this 1 hour ago b10174 7be2c65 model: add NextN/MTP speculative decoding support for GLM_DSA (GLM-5.2) (#25980)model: add NextN/MTP speculative decoding support for GLM_DSA (GLM-5.2)Adds GLM-5.2 NextN/MTP as a --spec-type draft-mtp target: nextn tensorloading via the qwen35moe/step35-style presence probe, a graph_mtpbuilder (enorm/hnorm/eh_proj + dense MLA + sigmoid-gated MoE withshared expert + shared head with fallbacks, _s scale tensors passedfor NVFP4), t_h_nextn extraction in the trunk graph, and MTP-contextKV setup: the draft head runs dense MLA, so the MTP context uses aplain attention KV cache holding only the nextn layer(s) (samepattern as the hybrid Qwen3.5 MTP context) while the main contextkeeps the DSA cache, now filtered to trunk layers only.Co-Authored-By: Claude Fable 5 ***@***.*** : support --mtp/--no-mtp export for GlmMoeDsaForCausalLM (GLM-5.2)Opt GLM-5.2 into the supports_mtp_export contract (post-#25641 shape,mirroring HYV3Model/Step35Model): --no-mtp drops the appended NextNblock (blk.78) and its nextn_predict_layers KV; --mtp keeps only theNextN block plus shared embeddings/norm/lm_head. Default (bundled)output is unchanged.Co-Authored-By: Claude Fable 5 ***@***.***: Claude Fable 5 ***@***.***:https://llama.appmacOS/iOS:macOS Apple Silicon (arm64)macOS Apple Silicon (arm64, KleidiAI enabled) DISABLEDmacOS Intel (x64)iOS XCFrameworkLinux:Ubuntu x64 (CPU)Ubuntu arm64 (CPU)""" if __name__ == "__main__": verifyCooling(TEXT) for i in range(5): # break test(TEXT) First Bad Commit *No response* Relevant log output Logs 0.00.032.796 I cmn common_param: common_params_print_info: build 10175 (60bccc3) with Clang 20.1.8 for Windows x86_640.00.032.799 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) 0.00.032.801 I cmn common_param: device_info: 0.00.032.805 I cmn common_param: - CPU : Intel(R) Core(TM) i5-10400 CPU @ 2.90GHz (8018 MiB, 2663 MiB free) 0.00.032.847 I cmn common_param: system_info: n_threads = 1 (n_threads_batch = 1) / 12 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 0.00.032.851 I srv llama_server: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true 0.00.032.891 I srv init: running without SSL 0.00.032.975 I srv init: using 11 threads for HTTP server 0.00.033.267 W srv llama_server: ----------------- 0.00.033.268 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set 0.00.033.269 W srv llama_server: this can be a security risk (cross-origin attacks) 0.00.033.270 W srv llama_server: more info: #25655 0.00.033.270 W srv llama_server: ----------------- 0.00.033.296 I srv start: binding port with default address family 0.00.042.004 I srv load_model: loading model 'G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf' 0.00.042.008 I srv load_model: local path 'G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf' 0.00.101.464 I llama_model_loader: loaded meta data with 38 key-value pairs and 310 tensors from G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf (version GGUF V3 (latest)) 0.00.101.480 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. 0.00.101.488 I llama_model_loader: - kv 0: general.architecture str = qwen3 0.00.101.489 I llama_model_loader: - kv 1: general.type str = model 0.00.101.493 I llama_model_loader: - kv 2: general.name str = Jina Embeddings v5 Text Small Retrieval 0.00.101.495 I llama_model_loader: - kv 3: general.finetune str = retrieval 0.00.101.496 I llama_model_loader: - kv 4: general.basename str = jina-embeddings-v5-text 0.00.101.497 I llama_model_loader: - kv 5: general.size_label str = small 0.00.101.498 I llama_model_loader: - kv 6: general.license str = cc 0.00.101.501 I llama_model_loader: - kv 7: general.base_model.count u32 = 1 0.00.101.502 I llama_model_loader: - kv 8: general.base_model.0.name str = Qwen3 0.6B 0.00.101.503 I llama_model_loader: - kv 9: general.base_model.0.organization str = Qwen 0.00.101.505 I llama_model_loader: - kv 10: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3-0.6B 0.00.101.507 I llama_model_loader: - kv 11: qwen3.block_count u32 = 28 0.00.101.508 I llama_model_loader: - kv 12: qwen3.context_length u32 = 40960 0.00.101.509 I llama_model_loader: - kv 13: qwen3.embedding_length u32 = 1024 0.00.101.509 I llama_model_loader: - kv 14: qwen3.feed_forward_length u32 = 3072 0.00.101.510 I llama_model_loader: - kv 15: qwen3.attention.head_count u32 = 16 0.00.101.511 I llama_model_loader: - kv 16: qwen3.attention.head_count_kv u32 = 8 0.00.101.517 I llama_model_loader: - kv 17: qwen3.rope.freq_base f32 = 3500000.000000 0.00.101.519 I llama_model_loader: - kv 18: qwen3.attention.layer_norm_rms_epsilon f32 = 0.000001 0.00.101.520 I llama_model_loader: - kv 19: qwen3.attention.key_length u32 = 128 0.00.101.521 I llama_model_loader: - kv 20: qwen3.attention.value_length u32 = 128 0.00.101.522 I llama_model_loader: - kv 21: tokenizer.ggml.model str = gpt2 0.00.101.522 I llama_model_loader: - kv 22: tokenizer.ggml.pre str = qwen2 0.00.139.665 I llama_model_loader: - kv 23: tokenizer.ggml.tokens arr[str,151936] = ["!", "\"", "#", "$", "%", "&", "'", ... 0.00.151.009 I llama_model_loader: - kv 24: tokenizer.ggml.token_type arr[i32,151936] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... 0.00.189.852 I llama_model_loader: - kv 25: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... 0.00.189.857 I llama_model_loader: - kv 26: tokenizer.ggml.eos_token_id u32 = 151645 0.00.189.858 I llama_model_loader: - kv 27: tokenizer.ggml.padding_token_id u32 = 151643 0.00.189.859 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 151643 0.00.189.860 I llama_model_loader: - kv 29: tokenizer.ggml.add_bos_token bool = false 0.00.189.866 I llama_model_loader: - kv 30: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>... 0.00.189.867 I llama_model_loader: - kv 31: general.quantization_version u32 = 2 0.00.189.868 I llama_model_loader: - kv 32: general.file_type u32 = 15 0.00.189.870 I llama_model_loader: - kv 33: quantize.imatrix.file str = /home/mguenther/multimodal-large-scal... 0.00.189.871 I llama_model_loader: - kv 34: quantize.imatrix.dataset str = data/calibration_data_v5_rc.txt 0.00.189.872 I llama_model_loader: - kv 35: quantize.imatrix.entries_count u32 = 196 0.00.189.873 I llama_model_loader: - kv 36: quantize.imatrix.chunks_count u32 = 225 0.00.189.874 I llama_model_loader: - kv 37: qwen3.pooling_type u32 = 3 0.00.189.875 I llama_model_loader: - type f32: 113 tensors 0.00.189.875 I llama_model_loader: - type q4_K: 168 tensors 0.00.189.876 I llama_model_loader: - type q6_K: 29 tensors 0.00.189.881 I print_info: file format = GGUF V3 (latest) 0.00.189.881 I print_info: file type = Q4_K - Medium 0.00.189.885 I print_info: file size = 372.65 MiB (5.24 BPW) 0.00.302.760 I load: 0 unused tokens 0.00.322.104 W load: control-looking token: 128247 '</s>' was not control-type; this is probably a bug in the model. its type will be overridden 0.00.327.844 I load: printing all EOG tokens: 0.00.327.847 I load: - 128247 ('</s>') 0.00.327.847 I load: - 151643 ('<|endoftext|>') 0.00.327.848 I load: - 151645 ('<|im_end|>') 0.00.327.848 I load: - 151662 ('<|fim_pad|>') 0.00.327.848 I load: - 151663 ('<|repo_name|>') 0.00.327.849 I load: - 151664 ('<|file_sep|>') 0.00.328.339 I load: special tokens cache size = 27 0.00.368.578 I load: token to piece cache size = 0.9311 MB 0.00.368.606 I print_info: arch = qwen3 0.00.368.608 I print_info: vocab_only = 0 0.00.368.608 I print_info: no_alloc = 0 0.00.368.608 I print_info: n_ctx_train = 40960 0.00.368.609 I print_info: n_embd_inp = 1024 0.00.368.609 I print_info: n_embd = 1024 0.00.368.610 I print_info: n_embd_out = 1024 0.00.368.610 I print_info: n_layer = 28 0.00.368.610 I print_info: n_layer_all = 28 0.00.368.638 I print_info: n_head = 16 0.00.368.646 I print_info: n_head_kv = 8 0.00.368.647 I print_info: n_rot = 128 0.00.368.647 I print_info: n_swa = 0 0.00.368.647 I print_info: is_swa_any = 0 0.00.368.648 I print_info: n_embd_head_k = 128 0.00.368.648 I print_info: n_embd_head_v = 128 0.00.368.655 I print_info: n_gqa = 2 0.00.368.662 I print_info: n_embd_k_gqa = 1024 0.00.368.678 I print_info: n_embd_v_gqa = 1024 0.00.368.683 I print_info: f_norm_eps = 0.0e+00 0.00.368.684 I print_info: f_norm_rms_eps = 1.0e-06 0.00.368.684 I print_info: f_clamp_kqv = 0.0e+00 0.00.368.685 I print_info: f_max_alibi_bias = 0.0e+00 0.00.368.685 I print_info: f_logit_scale = 0.0e+00 0.00.368.686 I print_info: f_attn_scale = 0.0e+00 0.00.368.686 I print_info: f_attn_value_scale = 0.0000 0.00.368.693 I print_info: n_ff = 3072 0.00.368.694 I print_info: n_expert = 0 0.00.368.694 I print_info: n_expert_used = 0 0.00.368.695 I print_info: n_expert_groups = 0 0.00.368.695 I print_info: n_group_used = 0 0.00.368.695 I print_info: causal attn = 1 0.00.368.696 I print_info: pooling type = 3 0.00.368.696 I print_info: rope type = 2 0.00.368.696 I print_info: rope scaling = linear 0.00.368.698 I print_info: freq_base_train = 3500000.0 0.00.368.699 I print_info: freq_scale_train = 1 0.00.368.699 I print_info: n_ctx_orig_yarn = 40960 0.00.368.700 I print_info: rope_yarn_log_mul = 0.0000 0.00.368.700 I print_info: rope_finetuned = unknown 0.00.368.701 I print_info: model type = 0.6B 0.00.368.703 I print_info: model params = 596.05 M 0.00.368.703 I print_info: general.name = Jina Embeddings v5 Text Small Retrieval 0.00.368.704 I print_info: vocab type = BPE 0.00.368.704 I print_info: n_vocab = 151936 0.00.368.705 I print_info: n_merges = 151387 0.00.368.705 I print_info: BOS token = 151643 '<|endoftext|>' 0.00.368.706 I print_info: EOS token = 151645 '<|im_end|>' 0.00.368.706 I print_info: EOT token = 151645 '<|im_end|>' 0.00.368.707 I print_info: PAD token = 151643 '<|endoftext|>' 0.00.368.707 I print_info: LF token = 198 'Ċ' 0.00.368.708 I print_info: FIM PRE token = 151659 '<|fim_prefix|>' 0.00.368.708 I print_info: FIM SUF token = 151661 '<|fim_suffix|>' 0.00.368.709 I print_info: FIM MID token = 151660 '<|fim_middle|>' 0.00.368.709 I print_info: FIM PAD token = 151662 '<|fim_pad|>' 0.00.368.709 I print_info: FIM REP token = 151663 '<|repo_name|>' 0.00.368.710 I print_info: FIM SEP token = 151664 '<|file_sep|>' 0.00.368.710 I print_info: EOG token = 128247 '</s>' 0.00.368.711 I print_info: EOG token = 151643 '<|endoftext|>' 0.00.368.711 I print_info: EOG token = 151645 '<|im_end|>' 0.00.368.712 I print_info: EOG token = 151662 '<|fim_pad|>' 0.00.368.712 I print_info: EOG token = 151663 '<|repo_name|>' 0.00.368.713 I print_info: EOG token = 151664 '<|file_sep|>' 0.00.368.713 I print_info: max token length = 256 0.00.368.715 I load_tensors: loading model tensors, this can take a while... (load_mode = dio) 0.00.382.521 I load_tensors: offloading output layer to GPU 0.00.382.522 I load_tensors: offloading 27 repeating layers to GPU 0.00.382.522 I load_tensors: offloaded 29/29 layers to GPU 0.00.382.526 I load_tensors: CPU model buffer size = 167.90 MiB 0.00.382.527 I load_tensors: CPU_REPACK model buffer size = 204.75 MiB 0.00.606.909 I cmn common_init_: added </s> logit bias = -inf 0.00.607.078 I cmn common_init_: added <|endoftext|> logit bias = -inf 0.00.607.087 I cmn common_init_: added <|im_end|> logit bias = -inf 0.00.607.088 I cmn common_init_: added <|fim_pad|> logit bias = -inf 0.00.607.089 I cmn common_init_: added <|repo_name|> logit bias = -inf 0.00.607.089 I cmn common_init_: added <|file_sep|> logit bias = -inf 0.00.607.763 I llama_context: constructing llama_context 0.00.607.769 I llama_context: n_seq_max = 4 0.00.607.769 I llama_context: n_ctx = 2048 0.00.607.770 I llama_context: n_ctx_seq = 2048 0.00.607.771 I llama_context: n_batch = 64 0.00.607.771 I llama_context: n_ubatch = 64 0.00.607.771 I llama_context: causal_attn = 1 0.00.607.772 I llama_context: flash_attn = disabled 0.00.607.772 I llama_context: kv_unified = true 0.00.607.776 I llama_context: freq_base = 3500000.0 0.00.607.777 I llama_context: freq_scale = 1 0.00.607.777 I llama_context: n_rs_seq = 0 0.00.607.777 I llama_context: n_outputs_max = 64 0.00.607.778 I llama_context: n_ctx_seq (2048) < n_ctx_train (40960) -- the full capacity of the model will not be utilized 0.00.608.159 I llama_context: CPU output buffer size = 2.33 MiB 0.00.608.318 I llama_kv_cache: CPU KV buffer size = 224.00 MiB 0.00.646.316 I llama_kv_cache: size = 224.00 MiB ( 2048 cells, 28 layers, 4/1 seqs), K (f16): 112.00 MiB, V (f16): 112.00 MiB 0.00.646.330 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 128 0.00.646.330 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 128 0.00.646.348 I sched_reserve: reserving ... 0.00.647.727 I resolve_fused_ops: resolving fused Gated Delta Net support: 0.00.648.316 I resolve_fused_ops: fused Gated Delta Net (autoregressive) enabled 0.00.648.804 I resolve_fused_ops: fused Gated Delta Net (chunked) enabled 0.00.648.805 I resolve_fused_ops: resolving fused Lightning Indexer support: 0.00.649.274 I resolve_fused_ops: Lightning Indexer enabled 0.00.649.274 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: 0.00.649.713 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled 0.00.650.144 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled 0.00.650.570 I resolve_fused_ops: fused DeepSeek V4 HC post enabled 0.00.652.833 I sched_reserve: CPU compute buffer size = 38.35 MiB 0.00.652.838 I sched_reserve: graph nodes = 1127 0.00.652.839 I sched_reserve: graph splits = 1 0.00.652.841 I sched_reserve: reserve took 6.49 ms, sched copies = 1 0.00.653.396 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) 0.00.827.519 I srv load_model: initializing, n_slots = 4, n_ctx_slot = 2048, kv_unified = 'true' 0.00.827.555 I spec common_specu: no implementations specified for speculative decoding 0.00.827.558 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048 0.00.827.562 I slot load_model: id 1 | task -1 | new slot, n_ctx = 2048 0.00.827.562 I slot load_model: id 2 | task -1 | new slot, n_ctx = 2048 0.00.827.563 I slot load_model: id 3 | task -1 | new slot, n_ctx = 2048 0.00.827.581 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it 0.00.827.582 I srv load_model: for more info see #16391 0.00.827.585 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 0.00.827.629 W srv init: --cache-idle-slots requires --cache-ram, disabling 0.00.847.574 I srv init: init: chat template, example_format: '<|im_start|>system You are a helpful assistant<|im_end|> <|im_start|>user Hello<|im_end|> <|im_start|>assistant Hi there<|im_end|> <|im_start|>user How are you?<|im_end|> <|im_start|>assistant ' 0.00.858.907 I srv init: init: chat template, thinking = 1 0.00.858.936 I srv llama_server: model loaded 0.00.858.938 I srv llama_server: listening on http://0.0.0.0:8082 0.00.858.945 I srv update_slots: all slots are idle 0.02.702.599 I slot get_availabl: id 3 | task -1 | selected slot by LRU, t_last = -1 0.02.702.611 I slot launch_slot_: id 3 | task 0 | processing task, is_child = 0 0.02.702.624 I slot operator(): id 3 | task 0 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 502 0.02.702.630 I slot operator(): id 3 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) 0.03.883.046 I slot operator(): id 3 | task 0 | cached n_tokens = 64, memory_seq_rm [64, end) 0.05.054.032 I slot operator(): id 3 | task 0 | cached n_tokens = 128, memory_seq_rm [128, end) 0.06.218.128 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 192, progress = 0.38, t = 3.52 s / 54.62 tokens per second 0.06.218.132 I slot operator(): id 3 | task 0 | cached n_tokens = 192, memory_seq_rm [192, end) 0.07.385.030 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 256, progress = 0.51, t = 4.68 s / 54.67 tokens per second 0.07.385.034 I slot operator(): id 3 | task 0 | cached n_tokens = 256, memory_seq_rm [256, end) 0.08.631.066 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 320, progress = 0.64, t = 5.93 s / 53.98 tokens per second 0.08.631.071 I slot operator(): id 3 | task 0 | cached n_tokens = 320, memory_seq_rm [320, end) 0.09.872.124 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 384, progress = 0.76, t = 7.17 s / 53.56 tokens per second 0.09.872.128 I slot operator(): id 3 | task 0 | cached n_tokens = 384, memory_seq_rm [384, end) 0.11.107.041 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 448, progress = 0.89, t = 8.40 s / 53.31 tokens per second 0.11.107.045 I slot operator(): id 3 | task 0 | cached n_tokens = 448, memory_seq_rm [448, end) 0.12.166.493 I slot release: id 3 | task 0 | stop processing: n_tokens = 502, truncated = 0 0.12.166.503 I srv update_slots: all slots are idle 0.12.169.820 I slot get_availabl: id 2 | task -1 | selected slot by LRU, t_last = -1 0.12.169.826 I slot launch_slot_: id 2 | task 9 | processing task, is_child = 0 0.12.169.832 I slot operator(): id 2 | task 9 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398 0.12.169.836 I slot operator(): id 2 | task 9 | cached n_tokens = 0, memory_seq_rm [0, end) 0.13.505.340 I slot operator(): id 2 | task 9 | cached n_tokens = 64, memory_seq_rm [64, end) 0.14.820.498 I slot operator(): id 2 | task 9 | cached n_tokens = 128, memory_seq_rm [128, end) 0.16.143.926 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 192, progress = 0.48, t = 3.97 s / 48.31 tokens per second 0.16.143.931 I slot operator(): id 2 | task 9 | cached n_tokens = 192, memory_seq_rm [192, end) 0.17.459.193 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 256, progress = 0.64, t = 5.29 s / 48.40 tokens per second 0.17.459.197 I slot operator(): id 2 | task 9 | cached n_tokens = 256, memory_seq_rm [256, end) 0.18.856.102 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 320, progress = 0.80, t = 6.69 s / 47.86 tokens per second 0.18.856.106 I slot operator(): id 2 | task 9 | cached n_tokens = 320, memory_seq_rm [320, end) 0.20.280.738 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 384, progress = 0.96, t = 8.11 s / 47.34 tokens per second 0.20.280.743 I slot operator(): id 2 | task 9 | cached n_tokens = 384, memory_seq_rm [384, end) 0.20.671.692 I slot release: id 2 | task 9 | stop processing: n_tokens = 398, truncated = 0 0.20.671.740 I srv update_slots: all slots are idle 0.20.677.081 I slot get_availabl: id 1 | task -1 | selected slot by LRU, t_last = -1 0.20.677.088 I slot launch_slot_: id 1 | task 17 | processing task, is_child = 0 0.20.677.096 I slot operator(): id 1 | task 17 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398 0.20.677.099 I slot operator(): id 1 | task 17 | cached n_tokens = 0, memory_seq_rm [0, end) 0.22.121.895 I slot operator(): id 1 | task 17 | cached n_tokens = 64, memory_seq_rm [64, end) 0.23.653.748 I slot operator(): id 1 | task 17 | cached n_tokens = 128, memory_seq_rm [128, end) 0.25.195.515 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 192, progress = 0.48, t = 4.52 s / 42.49 tokens per second 0.25.195.519 I slot operator(): id 1 | task 17 | cached n_tokens = 192, memory_seq_rm [192, end) 0.26.750.740 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 256, progress = 0.64, t = 6.07 s / 42.15 tokens per second 0.26.750.744 I slot operator(): id 1 | task 17 | cached n_tokens = 256, memory_seq_rm [256, end) 0.28.267.570 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 320, progress = 0.80, t = 7.59 s / 42.16 tokens per second 0.28.267.574 I slot operator(): id 1 | task 17 | cached n_tokens = 320, memory_seq_rm [320, end) 0.29.939.111 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 384, progress = 0.96, t = 9.26 s / 41.46 tokens per second 0.29.939.116 I slot operator(): id 1 | task 17 | cached n_tokens = 384, memory_seq_rm [384, end) 0.30.358.747 I slot release: id 1 | task 17 | stop processing: n_tokens = 398, truncated = 0 0.30.358.758 I srv update_slots: all slots are idle 0.30.361.726 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 0.30.361.734 I slot launch_slot_: id 0 | task 25 | processing task, is_child = 0 0.30.361.741 I slot operator(): id 0 | task 25 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398 0.30.361.744 I slot operator(): id 0 | task 25 | cached n_tokens = 0, memory_seq_rm [0, end) 0.31.975.433 I slot operator(): id 0 | task 25 | cached n_tokens = 64, memory_seq_rm [64, end) 0.33.600.319 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 128, progress = 0.32, t = 3.24 s / 39.52 tokens per second 0.33.600.324 I slot operator(): id 0 | task 25 | cached n_tokens = 128, memory_seq_rm [128, end) 0.35.227.110 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 192, progress = 0.48, t = 4.87 s / 39.46 tokens per second 0.35.227.114 I slot operator(): id 0 | task 25 | cached n_tokens = 192, memory_seq_rm [192, end) 0.36.945.258 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 256, progress = 0.64, t = 6.58 s / 38.89 tokens per second 0.36.945.262 I slot operator(): id 0 | task 25 | cached n_tokens = 256, memory_seq_rm [256, end) 0.38.611.448 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 320, progress = 0.80, t = 8.25 s / 38.79 tokens per second 0.38.611.453 I slot operator(): id 0 | task 25 | cached n_tokens = 320, memory_seq_rm [320, end) 0.40.345.882 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 384, progress = 0.96, t = 9.98 s / 38.46 tokens per second 0.40.345.887 I slot operator(): id 0 | task 25 | cached n_tokens = 384, memory_seq_rm [384, end) 0.40.781.556 I slot release: id 0 | task 25 | stop processing: n_tokens = 398, truncated = 0 0.40.781.566 I srv update_slots: all slots are idle — Reply to this email directly, view it on GitHub <#26282?email_source=notifications&email_token=CI26CIGKVCMYNKY6CAV6TO35HIGS3A5CNFSL4Z3JMQ5C6L3HNF2C22DVMIXUS43TOVSS6NJQGEYDMOBZGAZDTJTSMVQXG33OVJZXKYTTMNZGSYTFMSSWK5TFNZ2KYZTPN52GK4S7MNWGSY3L>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/CI26CIDVGUWQXIVLFKOWMYD5HIGS3AVCNFSNUABFKJSXA33TNF2G64TZHM3DCMRTGU2DOOBUHNEXG43VMU5TKMBRGA3DQOJQGI42C5QC> . Triage notifications, keep track of coding agent tasks and review pull requests on the go with GitHub Mobile for iOS <https://github.com/notifications/mobile/ios/CI26CIHWASBBR7HIPBKTS735HIGS3A5CNFSL4Z3JMQ5C6L3HNF2C22DVMIXUS43TOVSS6NJQGEYDMOBZGAZDTJTSMVQXG33OVJZXKYTTMNZGSYTFMSSWK5TFNZ2KUZTPN52GK4S7NFXXG> and Android <https://github.com/notifications/mobile/android/CI26CIHNFVVE6JHMERZAOZT5HIGS3A5CNFSL4Z3JMQ5C6L3HNF2C22DVMIXUS43TOVSS6NJQGEYDMOBZGAZDTJTSMVQXG33OVJZXKYTTMNZGSYTFMSSWK5TFNZ2K4ZTPN52GK4S7MFXGI4TPNFSA>. Download it today! You are receiving this because you are subscribed to this thread.Message ID: ***@***.***>

[ffsgsgss4cs75s]

Agreed. Running the same functional test suite against both SQLite and Postgres is a good way to catch portability issues early. Keeping migration tests separate also helps avoid accidental reliance on SQLite-specific behavior.

…

On Wed, 29 Jul 2026 at 15:55, zzffesgg58 ***@***.***> wrote: *zzffesgg58* left a comment (ggml-org/llama.cpp#26282) <#26282 (comment)> Thanks for the clarification. Understood. I'll just open a PR if I have a fix rather than asking for assignment. If someone beats me to it, that's perfectly fine. On Wed, Jul 29, 2026 at 3:01 PM fgst666 ***@***.***> wrote: > *fgst666* created an issue (ggml-org/llama.cpp#26282) > <#26282> > Name and Version > > build 10175 (60bccc3 > < 60bccc3>) > with Clang 20.1.8 for Windows x86_64 > Operating systems > > Windows > GGML backends > > CPU > Hardware > > i5 10400 > Models > > jina-embeddings-v5-nano-retrieval-Q5_K_M.gguf > Problem description & steps to reproduce > > llama-server --embedding got wrong data > Incorrect data output only happens after requests. > For the same text, the vector obtained from the first request after each > cold start is always exactly the same. > The bug doesn't produce completely wrong output — it causes a noticeable > quality drop, with an L2 distance of about 0.02 (for normalized vectors). > (This may depend on how much the cache has been polluted. just I think) > llama-server -m G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf --embedding > --pooling last -c 2048 -b 64 -ub 64 --load-mode dio --port 8082 --cache-ram > 0 -lv 4 -fa off --threads 1 -fit off > llama-server defaults to 4 slots with kvu enabled. When I manually > specify -np 4 (which defaults to -no-kvu), the bug does not occur. > > def verifyCooling(txt:str): > vecA = getVec(txt) > print("A:", vecA[:5]) > input("restart llama-server") > vecB = getVec(txt) > print("B:", vecB[:5]) > l2d = computeL2(vecA, vecB) > print("all cooling L2 =", l2d) > print() > > this fine, same, L2=0 , OK > > A: [-0.06865525245666504, 0.02405271865427494, -0.07321952283382416, -0.0781535804271698, 0.12095746397972107] > restart llama-server > B: [-0.06865525245666504, 0.02405271865427494, -0.07321952283382416, -0.0781535804271698, 0.12095746397972107] > all cooling L2 = 0.0 > > ------------------------------ > > but after another requests, input same text, got wrong > > def outCache(): > # for _ in range(5): > for _ in range(random.randint(5, 8)): > # t = f"{random.random():.5f}" * 20 > t = [f"{random.random():.5f}" for _ in range(random.randint(16, 24))] > _ = getVec(" ".join(t)) > def test(txt:str): > outCache() > vecA = getVec(txt) > print("A:", vecA[:5]) > outCache() > vecB = getVec(txt) > print("B:", vecB[:5]) > l2d = computeL2(vecA, vecB) > print(f"L2 = {l2d}\n") > > If the commented-out code is used instead, the results look like this: > > restart llama-server1 > B: [-0.06758762151002884, 0.0248175747692585, -0.07288508117198944, -0.07876278460025787, 0.12040942907333374] > all cooling L2 = 0.0 > > A: [-0.06671811640262604, 0.025068677961826324, -0.07316775619983673, -0.07915324717760086, 0.12086104601621628] > B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] > L2 = 0.025976544239568217 > > A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] > B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] > L2 = 0.025698335751558988 > > A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] > B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] > L2 = 0.025698335751558988 > > A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] > B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] > L2 = 0.025698335751558988 > > A: [-0.06698685139417648, 0.024658896028995514, -0.07315758615732193, -0.07918356359004974, 0.12041305005550385] > B: [-0.06759851425886154, 0.024288678541779518, -0.07291588187217712, -0.0785275548696518, 0.12057992815971375] > L2 = 0.025698335751558988 > > ------------------------------ > > all code > > import json, math, random > import requests > > def computeL2(a:list[float], b:list[float]): > return math.sqrt(sum((x-y)**2 for x,y in zip(a,b))) > > def getVec(t:str) -> list[float]: > body = { > "input": t, > "encoding_format": "float" > } > with requests.post("http://127.0.0.1:8082/v1/embeddings", json=body) as resp: > resp.raise_for_status() > rbody = json.loads(resp.content) > return rbody["data"][0]["embedding"] > > def outCache(): > # for _ in range(5): > for _ in range(random.randint(5, 8)): > # t = f"{random.random():.5f}" * 20 > t = [f"{random.random():.5f}" for _ in range(random.randint(16, 24))] > _ = getVec(" ".join(t)) > > def test(txt:str): > outCache() > vecA = getVec(txt) > print("A:", vecA[:5]) > outCache() > vecB = getVec(txt) > print("B:", vecB[:5]) > l2d = computeL2(vecA, vecB) > print(f"L2 = {l2d}\n") > > def verifyCooling(txt:str): > vecA = getVec(txt) > print("A:", vecA[:5]) > input("restart llama-server") > vecB = getVec(txt) > print("B:", vecB[:5]) > l2d = computeL2(vecA, vecB) > print("all cooling L2 =", l2d) > print() > > TEXT = ***@***.*** github-actions released this 1 hour ago b10174 7be2c65 model: add NextN/MTP speculative decoding support for GLM_DSA (GLM-5.2) (#25980)model: add NextN/MTP speculative decoding support for GLM_DSA (GLM-5.2)Adds GLM-5.2 NextN/MTP as a --spec-type draft-mtp target: nextn tensorloading via the qwen35moe/step35-style presence probe, a graph_mtpbuilder (enorm/hnorm/eh_proj + dense MLA + sigmoid-gated MoE withshared expert + shared head with fallbacks, _s scale tensors passedfor NVFP4), t_h_nextn extraction in the trunk graph, and MTP-contextKV setup: the draft head runs dense MLA, so the MTP context uses aplain attention KV cache holding only the nextn layer(s) (samepattern as the hybrid Qwen3.5 MTP context) while the main contextkeeps the DSA cache, now filtered to trunk layers only.Co-Authored-By: Claude Fable 5 ***@***.*** : support --mtp/--no-mtp export for GlmMoeDsaForCausalLM (GLM-5.2)Opt GLM-5.2 into the supports_mtp_export contract (post-#25641 shape,mirroring HYV3Model/Step35Model): --no-mtp drops the appended NextNblock (blk.78) and its nextn_predict_layers KV; --mtp keeps only theNextN block plus shared embeddings/norm/lm_head. Default (bundled)output is unchanged.Co-Authored-By: Claude Fable 5 ***@***.***: Claude Fable 5 ***@***.***:https://llama.appmacOS/iOS:macOS Apple Silicon (arm64)macOS Apple Silicon (arm64, KleidiAI enabled) DISABLEDmacOS Intel (x64)iOS XCFrameworkLinux:Ubuntu x64 (CPU)Ubuntu arm64 (CPU)""" > if __name__ == "__main__": > verifyCooling(TEXT) > for i in range(5): > # break > test(TEXT) > > First Bad Commit > > *No response* > Relevant log output Logs > > 0.00.032.796 I cmn common_param: common_params_print_info: build 10175 (60bccc3) with Clang 20.1.8 for Windows x86_640.00.032.799 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) > 0.00.032.801 I cmn common_param: device_info: > 0.00.032.805 I cmn common_param: - CPU : Intel(R) Core(TM) i5-10400 CPU @ 2.90GHz (8018 MiB, 2663 MiB free) > 0.00.032.847 I cmn common_param: system_info: n_threads = 1 (n_threads_batch = 1) / 12 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | > 0.00.032.851 I srv llama_server: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true > 0.00.032.891 I srv init: running without SSL > 0.00.032.975 I srv init: using 11 threads for HTTP server > 0.00.033.267 W srv llama_server: ----------------- > 0.00.033.268 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set > 0.00.033.269 W srv llama_server: this can be a security risk (cross-origin attacks) > 0.00.033.270 W srv llama_server: more info: #25655 > 0.00.033.270 W srv llama_server: ----------------- > 0.00.033.296 I srv start: binding port with default address family > 0.00.042.004 I srv load_model: loading model 'G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf' > 0.00.042.008 I srv load_model: local path 'G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf' > 0.00.101.464 I llama_model_loader: loaded meta data with 38 key-value pairs and 310 tensors from G:/ai/jina/jina-v5-small-retrieval-Q4_K_M.gguf (version GGUF V3 (latest)) > 0.00.101.480 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. > 0.00.101.488 I llama_model_loader: - kv 0: general.architecture str = qwen3 > 0.00.101.489 I llama_model_loader: - kv 1: general.type str = model > 0.00.101.493 I llama_model_loader: - kv 2: general.name str = Jina Embeddings v5 Text Small Retrieval > 0.00.101.495 I llama_model_loader: - kv 3: general.finetune str = retrieval > 0.00.101.496 I llama_model_loader: - kv 4: general.basename str = jina-embeddings-v5-text > 0.00.101.497 I llama_model_loader: - kv 5: general.size_label str = small > 0.00.101.498 I llama_model_loader: - kv 6: general.license str = cc > 0.00.101.501 I llama_model_loader: - kv 7: general.base_model.count u32 = 1 > 0.00.101.502 I llama_model_loader: - kv 8: general.base_model.0.name str = Qwen3 0.6B > 0.00.101.503 I llama_model_loader: - kv 9: general.base_model.0.organization str = Qwen > 0.00.101.505 I llama_model_loader: - kv 10: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3-0.6B > 0.00.101.507 I llama_model_loader: - kv 11: qwen3.block_count u32 = 28 > 0.00.101.508 I llama_model_loader: - kv 12: qwen3.context_length u32 = 40960 > 0.00.101.509 I llama_model_loader: - kv 13: qwen3.embedding_length u32 = 1024 > 0.00.101.509 I llama_model_loader: - kv 14: qwen3.feed_forward_length u32 = 3072 > 0.00.101.510 I llama_model_loader: - kv 15: qwen3.attention.head_count u32 = 16 > 0.00.101.511 I llama_model_loader: - kv 16: qwen3.attention.head_count_kv u32 = 8 > 0.00.101.517 I llama_model_loader: - kv 17: qwen3.rope.freq_base f32 = 3500000.000000 > 0.00.101.519 I llama_model_loader: - kv 18: qwen3.attention.layer_norm_rms_epsilon f32 = 0.000001 > 0.00.101.520 I llama_model_loader: - kv 19: qwen3.attention.key_length u32 = 128 > 0.00.101.521 I llama_model_loader: - kv 20: qwen3.attention.value_length u32 = 128 > 0.00.101.522 I llama_model_loader: - kv 21: tokenizer.ggml.model str = gpt2 > 0.00.101.522 I llama_model_loader: - kv 22: tokenizer.ggml.pre str = qwen2 > 0.00.139.665 I llama_model_loader: - kv 23: tokenizer.ggml.tokens arr[str,151936] = ["!", "\"", "#", "$", "%", "&", "'", ... > 0.00.151.009 I llama_model_loader: - kv 24: tokenizer.ggml.token_type arr[i32,151936] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... > 0.00.189.852 I llama_model_loader: - kv 25: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... > 0.00.189.857 I llama_model_loader: - kv 26: tokenizer.ggml.eos_token_id u32 = 151645 > 0.00.189.858 I llama_model_loader: - kv 27: tokenizer.ggml.padding_token_id u32 = 151643 > 0.00.189.859 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 151643 > 0.00.189.860 I llama_model_loader: - kv 29: tokenizer.ggml.add_bos_token bool = false > 0.00.189.866 I llama_model_loader: - kv 30: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>... > 0.00.189.867 I llama_model_loader: - kv 31: general.quantization_version u32 = 2 > 0.00.189.868 I llama_model_loader: - kv 32: general.file_type u32 = 15 > 0.00.189.870 I llama_model_loader: - kv 33: quantize.imatrix.file str = /home/mguenther/multimodal-large-scal... > 0.00.189.871 I llama_model_loader: - kv 34: quantize.imatrix.dataset str = data/calibration_data_v5_rc.txt > 0.00.189.872 I llama_model_loader: - kv 35: quantize.imatrix.entries_count u32 = 196 > 0.00.189.873 I llama_model_loader: - kv 36: quantize.imatrix.chunks_count u32 = 225 > 0.00.189.874 I llama_model_loader: - kv 37: qwen3.pooling_type u32 = 3 > 0.00.189.875 I llama_model_loader: - type f32: 113 tensors > 0.00.189.875 I llama_model_loader: - type q4_K: 168 tensors > 0.00.189.876 I llama_model_loader: - type q6_K: 29 tensors > 0.00.189.881 I print_info: file format = GGUF V3 (latest) > 0.00.189.881 I print_info: file type = Q4_K - Medium > 0.00.189.885 I print_info: file size = 372.65 MiB (5.24 BPW) > 0.00.302.760 I load: 0 unused tokens > 0.00.322.104 W load: control-looking token: 128247 '</s>' was not control-type; this is probably a bug in the model. its type will be overridden > 0.00.327.844 I load: printing all EOG tokens: > 0.00.327.847 I load: - 128247 ('</s>') > 0.00.327.847 I load: - 151643 ('<|endoftext|>') > 0.00.327.848 I load: - 151645 ('<|im_end|>') > 0.00.327.848 I load: - 151662 ('<|fim_pad|>') > 0.00.327.848 I load: - 151663 ('<|repo_name|>') > 0.00.327.849 I load: - 151664 ('<|file_sep|>') > 0.00.328.339 I load: special tokens cache size = 27 > 0.00.368.578 I load: token to piece cache size = 0.9311 MB > 0.00.368.606 I print_info: arch = qwen3 > 0.00.368.608 I print_info: vocab_only = 0 > 0.00.368.608 I print_info: no_alloc = 0 > 0.00.368.608 I print_info: n_ctx_train = 40960 > 0.00.368.609 I print_info: n_embd_inp = 1024 > 0.00.368.609 I print_info: n_embd = 1024 > 0.00.368.610 I print_info: n_embd_out = 1024 > 0.00.368.610 I print_info: n_layer = 28 > 0.00.368.610 I print_info: n_layer_all = 28 > 0.00.368.638 I print_info: n_head = 16 > 0.00.368.646 I print_info: n_head_kv = 8 > 0.00.368.647 I print_info: n_rot = 128 > 0.00.368.647 I print_info: n_swa = 0 > 0.00.368.647 I print_info: is_swa_any = 0 > 0.00.368.648 I print_info: n_embd_head_k = 128 > 0.00.368.648 I print_info: n_embd_head_v = 128 > 0.00.368.655 I print_info: n_gqa = 2 > 0.00.368.662 I print_info: n_embd_k_gqa = 1024 > 0.00.368.678 I print_info: n_embd_v_gqa = 1024 > 0.00.368.683 I print_info: f_norm_eps = 0.0e+00 > 0.00.368.684 I print_info: f_norm_rms_eps = 1.0e-06 > 0.00.368.684 I print_info: f_clamp_kqv = 0.0e+00 > 0.00.368.685 I print_info: f_max_alibi_bias = 0.0e+00 > 0.00.368.685 I print_info: f_logit_scale = 0.0e+00 > 0.00.368.686 I print_info: f_attn_scale = 0.0e+00 > 0.00.368.686 I print_info: f_attn_value_scale = 0.0000 > 0.00.368.693 I print_info: n_ff = 3072 > 0.00.368.694 I print_info: n_expert = 0 > 0.00.368.694 I print_info: n_expert_used = 0 > 0.00.368.695 I print_info: n_expert_groups = 0 > 0.00.368.695 I print_info: n_group_used = 0 > 0.00.368.695 I print_info: causal attn = 1 > 0.00.368.696 I print_info: pooling type = 3 > 0.00.368.696 I print_info: rope type = 2 > 0.00.368.696 I print_info: rope scaling = linear > 0.00.368.698 I print_info: freq_base_train = 3500000.0 > 0.00.368.699 I print_info: freq_scale_train = 1 > 0.00.368.699 I print_info: n_ctx_orig_yarn = 40960 > 0.00.368.700 I print_info: rope_yarn_log_mul = 0.0000 > 0.00.368.700 I print_info: rope_finetuned = unknown > 0.00.368.701 I print_info: model type = 0.6B > 0.00.368.703 I print_info: model params = 596.05 M > 0.00.368.703 I print_info: general.name = Jina Embeddings v5 Text Small Retrieval > 0.00.368.704 I print_info: vocab type = BPE > 0.00.368.704 I print_info: n_vocab = 151936 > 0.00.368.705 I print_info: n_merges = 151387 > 0.00.368.705 I print_info: BOS token = 151643 '<|endoftext|>' > 0.00.368.706 I print_info: EOS token = 151645 '<|im_end|>' > 0.00.368.706 I print_info: EOT token = 151645 '<|im_end|>' > 0.00.368.707 I print_info: PAD token = 151643 '<|endoftext|>' > 0.00.368.707 I print_info: LF token = 198 'Ċ' > 0.00.368.708 I print_info: FIM PRE token = 151659 '<|fim_prefix|>' > 0.00.368.708 I print_info: FIM SUF token = 151661 '<|fim_suffix|>' > 0.00.368.709 I print_info: FIM MID token = 151660 '<|fim_middle|>' > 0.00.368.709 I print_info: FIM PAD token = 151662 '<|fim_pad|>' > 0.00.368.709 I print_info: FIM REP token = 151663 '<|repo_name|>' > 0.00.368.710 I print_info: FIM SEP token = 151664 '<|file_sep|>' > 0.00.368.710 I print_info: EOG token = 128247 '</s>' > 0.00.368.711 I print_info: EOG token = 151643 '<|endoftext|>' > 0.00.368.711 I print_info: EOG token = 151645 '<|im_end|>' > 0.00.368.712 I print_info: EOG token = 151662 '<|fim_pad|>' > 0.00.368.712 I print_info: EOG token = 151663 '<|repo_name|>' > 0.00.368.713 I print_info: EOG token = 151664 '<|file_sep|>' > 0.00.368.713 I print_info: max token length = 256 > 0.00.368.715 I load_tensors: loading model tensors, this can take a while... (load_mode = dio) > 0.00.382.521 I load_tensors: offloading output layer to GPU > 0.00.382.522 I load_tensors: offloading 27 repeating layers to GPU > 0.00.382.522 I load_tensors: offloaded 29/29 layers to GPU > 0.00.382.526 I load_tensors: CPU model buffer size = 167.90 MiB > 0.00.382.527 I load_tensors: CPU_REPACK model buffer size = 204.75 MiB > 0.00.606.909 I cmn common_init_: added </s> logit bias = -inf > 0.00.607.078 I cmn common_init_: added <|endoftext|> logit bias = -inf > 0.00.607.087 I cmn common_init_: added <|im_end|> logit bias = -inf > 0.00.607.088 I cmn common_init_: added <|fim_pad|> logit bias = -inf > 0.00.607.089 I cmn common_init_: added <|repo_name|> logit bias = -inf > 0.00.607.089 I cmn common_init_: added <|file_sep|> logit bias = -inf > 0.00.607.763 I llama_context: constructing llama_context > 0.00.607.769 I llama_context: n_seq_max = 4 > 0.00.607.769 I llama_context: n_ctx = 2048 > 0.00.607.770 I llama_context: n_ctx_seq = 2048 > 0.00.607.771 I llama_context: n_batch = 64 > 0.00.607.771 I llama_context: n_ubatch = 64 > 0.00.607.771 I llama_context: causal_attn = 1 > 0.00.607.772 I llama_context: flash_attn = disabled > 0.00.607.772 I llama_context: kv_unified = true > 0.00.607.776 I llama_context: freq_base = 3500000.0 > 0.00.607.777 I llama_context: freq_scale = 1 > 0.00.607.777 I llama_context: n_rs_seq = 0 > 0.00.607.777 I llama_context: n_outputs_max = 64 > 0.00.607.778 I llama_context: n_ctx_seq (2048) < n_ctx_train (40960) -- the full capacity of the model will not be utilized > 0.00.608.159 I llama_context: CPU output buffer size = 2.33 MiB > 0.00.608.318 I llama_kv_cache: CPU KV buffer size = 224.00 MiB > 0.00.646.316 I llama_kv_cache: size = 224.00 MiB ( 2048 cells, 28 layers, 4/1 seqs), K (f16): 112.00 MiB, V (f16): 112.00 MiB > 0.00.646.330 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 128 > 0.00.646.330 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 128 > 0.00.646.348 I sched_reserve: reserving ... > 0.00.647.727 I resolve_fused_ops: resolving fused Gated Delta Net support: > 0.00.648.316 I resolve_fused_ops: fused Gated Delta Net (autoregressive) enabled > 0.00.648.804 I resolve_fused_ops: fused Gated Delta Net (chunked) enabled > 0.00.648.805 I resolve_fused_ops: resolving fused Lightning Indexer support: > 0.00.649.274 I resolve_fused_ops: Lightning Indexer enabled > 0.00.649.274 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: > 0.00.649.713 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled > 0.00.650.144 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled > 0.00.650.570 I resolve_fused_ops: fused DeepSeek V4 HC post enabled > 0.00.652.833 I sched_reserve: CPU compute buffer size = 38.35 MiB > 0.00.652.838 I sched_reserve: graph nodes = 1127 > 0.00.652.839 I sched_reserve: graph splits = 1 > 0.00.652.841 I sched_reserve: reserve took 6.49 ms, sched copies = 1 > 0.00.653.396 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) > 0.00.827.519 I srv load_model: initializing, n_slots = 4, n_ctx_slot = 2048, kv_unified = 'true' > 0.00.827.555 I spec common_specu: no implementations specified for speculative decoding > 0.00.827.558 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048 > 0.00.827.562 I slot load_model: id 1 | task -1 | new slot, n_ctx = 2048 > 0.00.827.562 I slot load_model: id 2 | task -1 | new slot, n_ctx = 2048 > 0.00.827.563 I slot load_model: id 3 | task -1 | new slot, n_ctx = 2048 > 0.00.827.581 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it > 0.00.827.582 I srv load_model: for more info see #16391 > 0.00.827.585 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 > 0.00.827.629 W srv init: --cache-idle-slots requires --cache-ram, disabling > 0.00.847.574 I srv init: init: chat template, example_format: '<|im_start|>system > You are a helpful assistant<|im_end|> > <|im_start|>user > Hello<|im_end|> > <|im_start|>assistant > Hi there<|im_end|> > <|im_start|>user > How are you?<|im_end|> > <|im_start|>assistant > ' > 0.00.858.907 I srv init: init: chat template, thinking = 1 > 0.00.858.936 I srv llama_server: model loaded > 0.00.858.938 I srv llama_server: listening on http://0.0.0.0:8082 > 0.00.858.945 I srv update_slots: all slots are idle > 0.02.702.599 I slot get_availabl: id 3 | task -1 | selected slot by LRU, t_last = -1 > 0.02.702.611 I slot launch_slot_: id 3 | task 0 | processing task, is_child = 0 > 0.02.702.624 I slot operator(): id 3 | task 0 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 502 > 0.02.702.630 I slot operator(): id 3 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) > 0.03.883.046 I slot operator(): id 3 | task 0 | cached n_tokens = 64, memory_seq_rm [64, end) > 0.05.054.032 I slot operator(): id 3 | task 0 | cached n_tokens = 128, memory_seq_rm [128, end) > 0.06.218.128 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 192, progress = 0.38, t = 3.52 s / 54.62 tokens per second > 0.06.218.132 I slot operator(): id 3 | task 0 | cached n_tokens = 192, memory_seq_rm [192, end) > 0.07.385.030 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 256, progress = 0.51, t = 4.68 s / 54.67 tokens per second > 0.07.385.034 I slot operator(): id 3 | task 0 | cached n_tokens = 256, memory_seq_rm [256, end) > 0.08.631.066 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 320, progress = 0.64, t = 5.93 s / 53.98 tokens per second > 0.08.631.071 I slot operator(): id 3 | task 0 | cached n_tokens = 320, memory_seq_rm [320, end) > 0.09.872.124 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 384, progress = 0.76, t = 7.17 s / 53.56 tokens per second > 0.09.872.128 I slot operator(): id 3 | task 0 | cached n_tokens = 384, memory_seq_rm [384, end) > 0.11.107.041 I slot print_timing: id 3 | task 0 | prompt processing, n_tokens = 448, progress = 0.89, t = 8.40 s / 53.31 tokens per second > 0.11.107.045 I slot operator(): id 3 | task 0 | cached n_tokens = 448, memory_seq_rm [448, end) > 0.12.166.493 I slot release: id 3 | task 0 | stop processing: n_tokens = 502, truncated = 0 > 0.12.166.503 I srv update_slots: all slots are idle > 0.12.169.820 I slot get_availabl: id 2 | task -1 | selected slot by LRU, t_last = -1 > 0.12.169.826 I slot launch_slot_: id 2 | task 9 | processing task, is_child = 0 > 0.12.169.832 I slot operator(): id 2 | task 9 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398 > 0.12.169.836 I slot operator(): id 2 | task 9 | cached n_tokens = 0, memory_seq_rm [0, end) > 0.13.505.340 I slot operator(): id 2 | task 9 | cached n_tokens = 64, memory_seq_rm [64, end) > 0.14.820.498 I slot operator(): id 2 | task 9 | cached n_tokens = 128, memory_seq_rm [128, end) > 0.16.143.926 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 192, progress = 0.48, t = 3.97 s / 48.31 tokens per second > 0.16.143.931 I slot operator(): id 2 | task 9 | cached n_tokens = 192, memory_seq_rm [192, end) > 0.17.459.193 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 256, progress = 0.64, t = 5.29 s / 48.40 tokens per second > 0.17.459.197 I slot operator(): id 2 | task 9 | cached n_tokens = 256, memory_seq_rm [256, end) > 0.18.856.102 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 320, progress = 0.80, t = 6.69 s / 47.86 tokens per second > 0.18.856.106 I slot operator(): id 2 | task 9 | cached n_tokens = 320, memory_seq_rm [320, end) > 0.20.280.738 I slot print_timing: id 2 | task 9 | prompt processing, n_tokens = 384, progress = 0.96, t = 8.11 s / 47.34 tokens per second > 0.20.280.743 I slot operator(): id 2 | task 9 | cached n_tokens = 384, memory_seq_rm [384, end) > 0.20.671.692 I slot release: id 2 | task 9 | stop processing: n_tokens = 398, truncated = 0 > 0.20.671.740 I srv update_slots: all slots are idle > 0.20.677.081 I slot get_availabl: id 1 | task -1 | selected slot by LRU, t_last = -1 > 0.20.677.088 I slot launch_slot_: id 1 | task 17 | processing task, is_child = 0 > 0.20.677.096 I slot operator(): id 1 | task 17 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398 > 0.20.677.099 I slot operator(): id 1 | task 17 | cached n_tokens = 0, memory_seq_rm [0, end) > 0.22.121.895 I slot operator(): id 1 | task 17 | cached n_tokens = 64, memory_seq_rm [64, end) > 0.23.653.748 I slot operator(): id 1 | task 17 | cached n_tokens = 128, memory_seq_rm [128, end) > 0.25.195.515 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 192, progress = 0.48, t = 4.52 s / 42.49 tokens per second > 0.25.195.519 I slot operator(): id 1 | task 17 | cached n_tokens = 192, memory_seq_rm [192, end) > 0.26.750.740 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 256, progress = 0.64, t = 6.07 s / 42.15 tokens per second > 0.26.750.744 I slot operator(): id 1 | task 17 | cached n_tokens = 256, memory_seq_rm [256, end) > 0.28.267.570 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 320, progress = 0.80, t = 7.59 s / 42.16 tokens per second > 0.28.267.574 I slot operator(): id 1 | task 17 | cached n_tokens = 320, memory_seq_rm [320, end) > 0.29.939.111 I slot print_timing: id 1 | task 17 | prompt processing, n_tokens = 384, progress = 0.96, t = 9.26 s / 41.46 tokens per second > 0.29.939.116 I slot operator(): id 1 | task 17 | cached n_tokens = 384, memory_seq_rm [384, end) > 0.30.358.747 I slot release: id 1 | task 17 | stop processing: n_tokens = 398, truncated = 0 > 0.30.358.758 I srv update_slots: all slots are idle > 0.30.361.726 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 > 0.30.361.734 I slot launch_slot_: id 0 | task 25 | processing task, is_child = 0 > 0.30.361.741 I slot operator(): id 0 | task 25 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 398 > 0.30.361.744 I slot operator(): id 0 | task 25 | cached n_tokens = 0, memory_seq_rm [0, end) > 0.31.975.433 I slot operator(): id 0 | task 25 | cached n_tokens = 64, memory_seq_rm [64, end) > 0.33.600.319 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 128, progress = 0.32, t = 3.24 s / 39.52 tokens per second > 0.33.600.324 I slot operator(): id 0 | task 25 | cached n_tokens = 128, memory_seq_rm [128, end) > 0.35.227.110 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 192, progress = 0.48, t = 4.87 s / 39.46 tokens per second > 0.35.227.114 I slot operator(): id 0 | task 25 | cached n_tokens = 192, memory_seq_rm [192, end) > 0.36.945.258 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 256, progress = 0.64, t = 6.58 s / 38.89 tokens per second > 0.36.945.262 I slot operator(): id 0 | task 25 | cached n_tokens = 256, memory_seq_rm [256, end) > 0.38.611.448 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 320, progress = 0.80, t = 8.25 s / 38.79 tokens per second > 0.38.611.453 I slot operator(): id 0 | task 25 | cached n_tokens = 320, memory_seq_rm [320, end) > 0.40.345.882 I slot print_timing: id 0 | task 25 | prompt processing, n_tokens = 384, progress = 0.96, t = 9.98 s / 38.46 tokens per second > 0.40.345.887 I slot operator(): id 0 | task 25 | cached n_tokens = 384, memory_seq_rm [384, end) > 0.40.781.556 I slot release: id 0 | task 25 | stop processing: n_tokens = 398, truncated = 0 > 0.40.781.566 I srv update_slots: all slots are idle > > — > Reply to this email directly, view it on GitHub > < #26282?email_source=notifications&email_token=CI26CIGKVCMYNKY6CAV6TO35HIGS3A5CNFSL4Z3JMQ5C6L3HNF2C22DVMIXUS43TOVSS6NJQGEYDMOBZGAZDTJTSMVQXG33OVJZXKYTTMNZGSYTFMSSWK5TFNZ2KYZTPN52GK4S7MNWGSY3L>, > or unsubscribe > < https://github.com/notifications/unsubscribe-auth/CI26CIDVGUWQXIVLFKOWMYD5HIGS3AVCNFSNUABFKJSXA33TNF2G64TZHM3DCMRTGU2DOOBUHNEXG43VMU5TKMBRGA3DQOJQGI42C5QC> > . > Triage notifications, keep track of coding agent tasks and review pull > requests on the go with GitHub Mobile for iOS > < https://github.com/notifications/mobile/ios/CI26CIHWASBBR7HIPBKTS735HIGS3A5CNFSL4Z3JMQ5C6L3HNF2C22DVMIXUS43TOVSS6NJQGEYDMOBZGAZDTJTSMVQXG33OVJZXKYTTMNZGSYTFMSSWK5TFNZ2KUZTPN52GK4S7NFXXG> > and Android > < https://github.com/notifications/mobile/android/CI26CIHNFVVE6JHMERZAOZT5HIGS3A5CNFSL4Z3JMQ5C6L3HNF2C22DVMIXUS43TOVSS6NJQGEYDMOBZGAZDTJTSMVQXG33OVJZXKYTTMNZGSYTFMSSWK5TFNZ2K4ZTPN52GK4S7MFXGI4TPNFSA>. > Download it today! > You are receiving this because you are subscribed to this thread.Message > ID: ***@***.***> > — Reply to this email directly, view it on GitHub <#26282?email_source=notifications&email_token=CI3MU35BJH6LQHT2ICWV7B35HINARA5CNFSNUABFM5UWIORPF5TWS5BNNB2WEL2JONZXKZKDN5WW2ZLOOQXTKMJSGAZDONRRGI3KM4TFMFZW63VKON2WE43DOJUWEZLEUVSXMZLOOSWGM33PORSXEX3DNRUWG2Y#issuecomment-5120276126>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/CI3MU334NEFFTAYGT2QRLID5HINARAVCNFSNUABFKJSXA33TNF2G64TZHM3DCMRTGU2DOOBUHNEXG43VMU5TKMBRGA3DQOJQGI42C5QC> . Triage notifications, keep track of coding agent tasks and review pull requests on the go with GitHub Mobile for iOS <https://github.com/notifications/mobile/ios/CI3MU34TR4YXIMWOIHJ2RN35HINARA5CNFSNUABFM5UWIORPF5TWS5BNNB2WEL2JONZXKZKDN5WW2ZLOOQXTKMJSGAZDONRRGI3KM4TFMFZW63VKON2WE43DOJUWEZLEUVSXMZLOOSVGM33PORSXEX3JN5ZQ> and Android <https://github.com/notifications/mobile/android/CI3MU37WENGIJSORTPAPS3L5HINARA5CNFSNUABFM5UWIORPF5TWS5BNNB2WEL2JONZXKZKDN5WW2ZLOOQXTKMJSGAZDONRRGI3KM4TFMFZW63VKON2WE43DOJUWEZLEUVSXMZLOOSXGM33PORSXEX3BNZSHE33JMQ>. Download it today! You are receiving this because you are subscribed to this thread.Message ID: ***@***.***>

[fgst666]

I omitted something. If random outCache, the result error will be random

A: [-0.06697256863117218, 0.024824809283018112, -0.07290279865264893, -0.07730302959680557, 0.12035828828811646]
B: [-0.06773943454027176, 0.025232231244444847, -0.07290451228618622, -0.07868174463510513, 0.11979556828737259]
L2 = 0.02481928335343234

A: [-0.0675453394651413, 0.02576484903693199, -0.07296554744243622, -0.07837744802236557, 0.12097106128931046]
B: [-0.06811017543077469, 0.023957695811986923, -0.07296797633171082, -0.0802837684750557, 0.12105036526918411]
L2 = 0.024943413864625254

A: [-0.0673549696803093, 0.023683663457632065, -0.07272692024707794, -0.07895853370428085, 0.12033463269472122]
B: [-0.06809037178754807, 0.024985061958432198, -0.07289434224367142, -0.07923208922147751, 0.11992903798818588]
L2 = 0.01864937776532161

A: [-0.06745133548974991, 0.023066731169819832, -0.0731501430273056, -0.07840745896100998, 0.12172918766736984]
B: [-0.06892392039299011, 0.023856397718191147, -0.07311639934778214, -0.0786793902516365, 0.11965367943048477]
L2 = 0.023756165739162435

A: [-0.06693866103887558, 0.02412593550980091, -0.07279331982135773, -0.0782846137881279, 0.11925137788057327]
B: [-0.06849687546491623, 0.025126377120614052, -0.07287871092557907, -0.07912635803222656, 0.12029147148132324]
L2 = 0.01781648801162291

[fgst666]

With -np n(n > 1) , the bug occurs regardless of whether -kvu or -no-kvu is used. With -np 1, there is no bug. The bug only occurs during batch requests; it does not occur when sending requests one at a time.

one at a time

one at a time request; restart llama-server
[0.018023231998085976, 0.03916475921869278, -0.06961634755134583, -0.09721372276544571]
[0.018023231998085976, 0.03916475921869278, -0.06961634755134583, -0.09721372276544571]
0.000000

[0.03135450929403305, 0.04723431169986725, -0.07342424988746643, -0.10441286861896515]
[0.03135450929403305, 0.04723431169986725, -0.07342424988746643, -0.10441286861896515]
0.000000

[0.0011982946889474988, 0.04206746816635132, -0.06046701595187187, -0.10686028748750687]
[0.0011982946889474988, 0.04206746816635132, -0.06046701595187187, -0.10686028748750687]
0.000000

[0.003411010140553117, 0.04486425966024399, -0.06757170706987381, -0.10917430371046066]
[0.003411010140553117, 0.04486425966024399, -0.06757170706987381, -0.10917430371046066]
0.000000

[0.013008386828005314, 0.04814746975898743, -0.03920096158981323, -0.004169429652392864]
[0.013008386828005314, 0.04814746975898743, -0.03920096158981323, -0.004169429652392864]
0.000000

[0.01115778461098671, 0.038917314261198044, -0.06643574684858322, -0.09988481551408768]
[0.01115778461098671, 0.038917314261198044, -0.06643574684858322, -0.09988481551408768]
0.000000

[0.03235718607902527, 0.03850385919213295, -0.06345568597316742, -0.09817690402269363]
[0.03235718607902527, 0.03850385919213295, -0.06345568597316742, -0.09817690402269363]
0.000000

[-0.00765168247744441, 0.054124344140291214, -0.07373200356960297, -0.11071944236755371]
[-0.00765168247744441, 0.054124344140291214, -0.07373200356960297, -0.11071944236755371]
0.000000

batch

batch request; restart llama-server
[0.01852107048034668, 0.037660859525203705, -0.06952348351478577, -0.09637128561735153]
[0.019432656466960907, 0.03941904753446579, -0.06996078044176102, -0.09569894522428513]
0.031023

[0.03184835985302925, 0.04744306579232216, -0.07382102310657501, -0.10468605905771255]
[0.03134389594197273, 0.04650656878948212, -0.07368311285972595, -0.10455699265003204]
0.012353

[0.0012283612741157413, 0.040760915726423264, -0.06076950952410698, -0.10799729079008102]
[0.0011982946889474988, 0.04206746816635132, -0.06046701595187187, -0.10686028748750687]
0.014421

[0.0033656239975243807, 0.04522394388914108, -0.0676015168428421, -0.1099436804652214]
[0.0036931338254362345, 0.04335281252861023, -0.06771247088909149, -0.11093813925981522]
0.011856

[0.013236620463430882, 0.0470874197781086, -0.03925417363643646, -0.0043660919182002544]
[0.013236620463430882, 0.0470874197781086, -0.03925417363643646, -0.0043660919182002544]
0.000000

[0.012159846723079681, 0.038447264581918716, -0.06657682359218597, -0.10057635605335236]
[0.012159846723079681, 0.038447264581918716, -0.06657682359218597, -0.10057635605335236]
0.000000

[0.03238912299275398, 0.03845714032649994, -0.06371766328811646, -0.0985211506485939]
[0.03238912299275398, 0.03845714032649994, -0.06371766328811646, -0.0985211506485939]
0.000000

[-0.007331434637308121, 0.05326775088906288, -0.07376736402511597, -0.11066930741071701]
[-0.00765168247744441, 0.054124344140291214, -0.07373200356960297, -0.11071944236755371]
0.011998

---

An L2 distance error of 0.03 from same text input may not seem large, but it should be treated as a serious bug. This is unexpected behavior and the upper bound of the error is unknown.

Moreover, since it doesn't cause a crash but rather a degradation in quality, it makes downstream debugging significantly harder.

python

import json, random, math

import requests

TEN_TOKEN = "const t=Symbol.for('preact-');"

def computeL2(a:list[float], b:list[float]):
    return math.sqrt(sum((x-y)**2 for x,y in zip(a,b)))


def getVec(t:str) -> list[float]:
    body = {
        "input": t,
        "encoding_format": "float"
    }
    with requests.post("http://127.0.0.1:8082/v1/embeddings", json=body) as resp:
        resp.raise_for_status()
        rbody = json.loads(resp.content)
        return rbody["data"][0]["embedding"]


def getVecs(lst:list[str]) -> list[list[float]]:
    return [getVec(t) for t in lst]


def getVecsOnceRequest(texts:list[str]) -> list[list[float]]:
    body = {
        "input": [t for t in texts],
        "encoding_format": "float"
    }
    with requests.post("http://192.168.1.105:8082/v1/embeddings", json=body) as resp:
        resp.raise_for_status()
        rbody = json.loads(resp.content)
    data:list[dict] = rbody["data"]
    assert len(data) == len(texts)
    data.sort(key=lambda x: x["index"])
    return [d["embedding"] for d in data]


def outCache():
    # 51 tokens, 61, 71, 81, 91, 101
    for i in range(6):
        t = f"{i} " + TEN_TOKEN * (i+5)
        _ = getVec(t)

def diff(aVecs:list[list[float]], bVecs:list[list[float]]):
    for a, b in zip(aVecs, bVecs):
        print(a[:4])
        print(b[:4])
        print(f"{computeL2(a,b):.6f}\n")

def main(texts:list[str]):
    # one at a time
    input("one at a time request; restart llama-server")
    aVecs = getVecs(texts)
    outCache()
    bVecs = getVecs(texts)
    diff(aVecs, bVecs)
    # batch
    input("batch request; restart llama-server")
    aVecs = getVecsOnceRequest(texts)
    outCache()
    bVecs = getVecsOnceRequest(texts)
    diff(aVecs, bVecs)


DATA = [
    # 97 tokens
    "a 3 5 7" + TEN_TOKEN * 9,
    # 259
    "b 3 5 7 9" + TEN_TOKEN * 25,
    # 136
    "c 3 5;" + TEN_TOKEN * 13,
    # 114
    "d 3;" + TEN_TOKEN * 11,
    # 94
    "e 3;" + TEN_TOKEN * 9,
    # 107
    "f 3 5 7" + TEN_TOKEN * 10,
    # 229
    "g 3 5 7 9" + TEN_TOKEN * 22,
    # 181
    "h" + TEN_TOKEN * 18
]

if __name__ == "__main__":
    main(DATA)