From 2b3eb5a83e39cbef4386fbe6c8f6e281428ebfe6 Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Sat, 18 Jul 2026 13:50:45 -0500 Subject: [PATCH 1/2] Add eigen_eval_loss: forward-only held-out cross-entropy A language model should be selected on held-out loss. That metric did not exist here, so iLambdaAi ranked checkpoints on a generate-and-parse rate instead -- and that rate could not see the model. Three checkpoints scored an IDENTICAL 60/300 on the greedy parse metric while their held-out loss spanned 3.679 to 1.751, a perplexity gap of 39.6 vs 5.76. Greedy decoding collapses onto frequency attractors regardless of how good the model underneath is, so the metric separates trained from untrained (0 vs 200) and then flatlines across every trained candidate. Deterministic is not the same as informative. The sample size compounded it: at n=30 a 20% rate carries a 95% interval of roughly 8%-39%, so an A/B whose arms differed by 14 points of accuracy read as "no difference". A loss gives one datapoint per window on a continuous scale -- 2000 windows where there were 30 binary trials. Forward only: no backward, no weight update, no requantise, no observer state, and g_model_age/g_training_samples untouched, so scoring a checkpoint never mutates it. Log-softmax subtracts the max before exponentiating; without that a large logit overflows float and returns inf, which would silently poison a mean taken over thousands of windows. Suite [47d] pins it against the property that an untrained model must score ln(vocab_size), since a near-uniform output distribution has that cross-entropy whatever the target is. Measured 7.004 against a predicted 7.011 at vocab=1109, and 2.016 against 2.079 on the vocab=8 fixture. One anchor catches a missing max-subtraction, a mis-indexed target, a softmax summed over the wrong axis, and sign errors at once. Also checks out-of-range targets return the -1 sentinel rather than reading past the logits array. Suite: 3168/3168 on the http+model build. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 16 ++++++++ src/ext_names.h | 1 + src/model_infer.c | 70 ++++++++++++++++++++++++++++++++ src/model_internal.h | 1 + tests/run_all_tests.sh | 15 +++++++ tests/test_eval_loss.sh | 90 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 193 insertions(+) create mode 100755 tests/test_eval_loss.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ebb482a..70167e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,22 @@ All notable changes to EigenScript are documented here. button geometry is now defined once (`_dialog_btn_rect`) instead of being recomputed by render and click separately. +### Added +- **`eigen_eval_loss` — forward-only held-out cross-entropy (model build).** + Scores one held-out window without a backward pass, weight update, + requantise, or observer write, so evaluating a checkpoint never mutates it. + This is the metric a language model should be *selected* on, and it did not + previously exist: iLambdaAi ranked checkpoints on a generated-and-parsed + rate at n=30, where a 20% score has a 95% interval roughly 8%-39%. That + metric could not distinguish two models whose held-out perplexity differed + by 7x (39.6 vs 5.76) — it scored both at an identical 60/300, because greedy + decoding collapses onto frequency attractors regardless of model quality. + A loss gives one datapoint per window on a continuous scale instead of 300 + binary trials. Pinned by suite `[47d]` against the property that an + untrained model must score ln(vocab_size) — measured 7.004 against a + predicted 7.011 at vocab=1109 — which simultaneously catches a missing + max-subtraction (overflow to inf), a mis-indexed target, and sign errors. + ### Changed - **The observer learning-rate gate is now two-channel (entropy *and* drift).** `observer_matrix_scale` throttled a weight matrix to 0.5x whenever its diff --git a/src/ext_names.h b/src/ext_names.h index c90517dd..148a5878 100644 --- a/src/ext_names.h +++ b/src/ext_names.h @@ -110,6 +110,7 @@ #define EIGS_MODEL_BUILTINS(X) \ X(eigen_model_loaded, builtin_eigen_model_loaded) \ X(eigen_generate, builtin_eigen_generate) \ + X(eigen_eval_loss, builtin_eigen_eval_loss) \ X(eigen_model_info, builtin_eigen_model_info) \ X(native_train_step_builtin, builtin_native_train_step) \ X(model_save_weights, builtin_model_save_weights) \ diff --git a/src/model_infer.c b/src/model_infer.c index 1a9acb95..733f6b56 100644 --- a/src/model_infer.c +++ b/src/model_infer.c @@ -586,6 +586,76 @@ Value* builtin_eigen_generate(Value *arg) { return result; } +Value* builtin_eigen_eval_loss(Value *arg) { + /* Input: [prompt_ids_list, target_id] + * Output: cross-entropy of target_id given the prompt, in nats. + * + * Held-out loss is the selection metric a language model should be ranked + * on, and this is the forward-only path that makes it computable. The task + * metrics (parse rate) sample 300 generations; this scores one point per + * WINDOW, so a few thousand windows give ~10x the samples on a continuous + * scale instead of a binary one. That difference is not academic: a night + * of A/B runs on the n=30 parse rate could not distinguish two arms whose + * accuracy differed by 14 points, because a pass/fail metric at n=30 has a + * 95% interval about 30 points wide. + * + * Forward only -- no backward, no weight update, no requantise, no observer + * state, and g_model_age/g_training_samples are untouched, so scoring a + * checkpoint never mutates it. */ + if (!arg || arg->type != VAL_LIST || arg->data.list.count < 2) { + fprintf(stderr, "eigen_eval_loss: requires [prompt_ids, target_id]\n"); + return make_num(-1.0); + } + Value *prompt_list = arg->data.list.items[0]; + if (prompt_list->type != VAL_LIST) { + fprintf(stderr, "eigen_eval_loss: prompt_ids must be a list\n"); + return make_num(-1.0); + } + if (arg->data.list.items[1]->type != VAL_NUM) { + fprintf(stderr, "eigen_eval_loss: target_id must be a number\n"); + return make_num(-1.0); + } + int target_id = (int)arg->data.list.items[1]->data.num; + + if (!g_model.loaded) return make_num(-1.0); + int vocab_size = g_model.config.vocab_size; + if (target_id < 0 || target_id >= vocab_size) { + fprintf(stderr, "eigen_eval_loss: target_id %d out of range [0,%d)\n", target_id, vocab_size); + return make_num(-1.0); + } + + int prompt_len = prompt_list->data.list.count; + if (prompt_len <= 0) { + fprintf(stderr, "eigen_eval_loss: prompt must be non-empty\n"); + return make_num(-1.0); + } + + int *prompt_ids = xcalloc(prompt_len, sizeof(int)); + for (int i = 0; i < prompt_len; i++) { + Value *v = prompt_list->data.list.items[i]; + int t = (v->type == VAL_NUM) ? (int)v->data.num : 0; + if (t < 0 || t >= vocab_size) t = 0; + prompt_ids[i] = t; + } + + float *logits = xcalloc(vocab_size, sizeof(float)); + native_forward(prompt_ids, prompt_len, &g_model, logits); + free(prompt_ids); + + /* Numerically stable log-softmax at the target: subtract the max before + * exponentiating, or a large logit overflows float and returns inf/nan -- + * which would silently poison the mean over thousands of windows. */ + float maxl = logits[0]; + for (int j = 1; j < vocab_size; j++) if (logits[j] > maxl) maxl = logits[j]; + double sum = 0.0; + for (int j = 0; j < vocab_size; j++) sum += exp((double)(logits[j] - maxl)); + double loss = -((double)(logits[target_id] - maxl) - log(sum)); + free(logits); + + if (isnan(loss) || isinf(loss)) return make_num(-1.0); + return make_num(loss); +} + Value* builtin_eigen_model_info(Value *arg) { (void)arg; char buf[512]; diff --git a/src/model_internal.h b/src/model_internal.h index aef4b58e..c55baaf8 100644 --- a/src/model_internal.h +++ b/src/model_internal.h @@ -220,6 +220,7 @@ Value* builtin_eigen_model_save_binary(Value *arg); Value* builtin_eigen_checkpoint_info(Value *arg); Value* builtin_eigen_generate(Value *arg); Value* builtin_eigen_model_loaded(Value *arg); +Value* builtin_eigen_eval_loss(Value *arg); Value* builtin_eigen_model_info(Value *arg); Value* builtin_native_train_step(Value *arg); diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index da1eab7c..0aa8a079 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -1890,6 +1890,21 @@ if ! echo "$MODEL_PROBE_OUT" | grep -q "undefined variable"; then echo " PASS: batched training path is gradient-identical to the per-position oracle" fi echo "" + + echo "[47d/47] eigen_eval_loss held-out cross-entropy (4 checks)" + EL_OUTPUT=$(bash "$TESTS_DIR/test_eval_loss.sh" 2>&1) + EL_PASS=$(echo "$EL_OUTPUT" | grep -c "PASS:" || true) + EL_FAIL=$(echo "$EL_OUTPUT" | grep -c "FAIL:" || true) + TOTAL=$((TOTAL + EL_PASS + EL_FAIL)) + PASS=$((PASS + EL_PASS)) + FAIL=$((FAIL + EL_FAIL)) + if [ "$EL_FAIL" -gt 0 ]; then + echo " FAIL: $EL_FAIL eigen_eval_loss check(s) failed" + echo "$EL_OUTPUT" | grep "FAIL:" | head -4 + else + echo " PASS: untrained cross-entropy sits at ln(vocab)" + fi + echo "" else echo "[47/47] Model roundtrip SKIPPED (binary built without EIGENSCRIPT_EXT_MODEL)" echo "" diff --git a/tests/test_eval_loss.sh b/tests/test_eval_loss.sh new file mode 100755 index 00000000..57d96eba --- /dev/null +++ b/tests/test_eval_loss.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# eigen_eval_loss: forward-only held-out cross-entropy. +# +# The property that pins this down without needing a trained model: an +# UNTRAINED model's output distribution is near-uniform over the vocabulary, +# so its cross-entropy must sit at ln(vocab_size) whatever the target is. The +# tiny fixture has vocab=8, so every loss must land near ln(8) = 2.0794. +# +# That single anchor catches the realistic failure modes at once -- a missing +# max-subtraction (overflow -> inf), summing the softmax over the wrong axis, +# indexing the wrong target, or returning logits instead of a loss would all +# miss ln(8). It also catches sign errors, since a cross-entropy is never +# negative. +# +# Runs only when the binary is built with EIGENSCRIPT_EXT_MODEL=1. + +set -u +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +EIGS="$TESTS_DIR/../src/eigenscript" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +fail() { echo " FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); } + +MODEL=/tmp/eigs_el_tiny.json +HARNESS=/tmp/eigs_el_harness.eigs +cleanup() { rm -f "$MODEL" "$HARNESS" /tmp/eigs_el_*.log; } +trap cleanup EXIT + +if ! "$EIGS" "$TESTS_DIR/gen_tiny_model.eigs" > "$MODEL" 2>/tmp/eigs_el_gen.log; then + fail "EL00 generate tiny model" "see /tmp/eigs_el_gen.log" + echo "EVALLOSS: 0 passed, 1 failed" + exit 1 +fi + +cat > "$HARNESS" </dev/null) +LOSSES=$(printf '%s\n' "$OUT" | sed -n 's/^L //p') + +if [ -z "$LOSSES" ]; then + fail "EL01 eigen_eval_loss returned values" "no output" + echo "EVALLOSS: $PASS passed, $((FAIL+1)) failed" + exit 1 +fi +ok "EL01 eigen_eval_loss returned a loss for every target" + +# All 8 losses within 0.35 nats of ln(8). The band is wide enough for the +# fixture's small random weights to tilt the distribution, tight enough that +# any structural error lands far outside it. +BAD=$(printf '%s\n' "$LOSSES" | awk ' + BEGIN { ln8 = 2.0794415; bad = 0 } + { d = $1 - ln8; if (d < 0) d = -d; if (d > 0.35) bad++ } + END { print bad }') +if [ "$BAD" = "0" ]; then + ok "EL02 untrained losses sit at ln(vocab) (all 8 within 0.35 of 2.0794)" +else + fail "EL02 untrained losses deviate from ln(vocab)" "$BAD of 8 outside band" + echo " losses: $(printf '%s ' $LOSSES)" +fi + +NONNEG=$(printf '%s\n' "$LOSSES" | awk '{ if ($1 < 0) n++ } END { print n+0 }') +if [ "$NONNEG" = "0" ]; then + ok "EL03 no negative cross-entropy" +else + fail "EL03 negative cross-entropy returned" "$NONNEG value(s) < 0" +fi + +OOR=$(printf '%s\n' "$OUT" | sed -n 's/^OOR //p') +NEG=$(printf '%s\n' "$OUT" | sed -n 's/^NEG //p') +if [ "${OOR%%.*}" = "-1" ] && [ "${NEG%%.*}" = "-1" ]; then + ok "EL04 out-of-range targets rejected with the -1 sentinel" +else + fail "EL04 out-of-range target not rejected" "target=999 -> '$OOR', target=-1 -> '$NEG'" +fi + +echo "EVALLOSS: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] From 6ce7d03307cf913b4552a7e6064e36f43b67b970 Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Sat, 18 Jul 2026 15:24:53 -0500 Subject: [PATCH 2/2] Add top-p (nucleus) sampling to eigen_generate Optional 4th argument to eigen_generate. Keeps the smallest set of tokens whose cumulative probability reaches p, so the candidate set adapts to the model's confidence, instead of the fixed top-k=40 that admits 39 near-zero tokens where the distribution is sharp and truncates real choices where it is flat. Absent or outside (0,1) keeps the historical top-k path, so existing callers and every recorded number are unchanged -- decoding policy is opt-in, not a silent default flip. Honest scope: this was written to fix a repetition degeneracy and it does not. The model assigns its repeat-loop continuation p=0.987 (loss 0.0127, alternatives at ~3e-8), so a 0.95 nucleus keeps exactly one token and produces output identical to top-k, token for token. The degeneracy is a genuine self-consistent fixed point in the model, not a decoding artifact. Nucleus sampling is still the better default policy and is worth having; it just is not the fix for that. Testing needed care. Generation draws from a global rand(), so two calls with the SAME policy already disagree on most positions -- measured 3/40 identical. Comparing token sequences therefore proves nothing about the policy, and an earlier version of this check "verified" top-p by observing 0/40 matches, which the control shows is indistinguishable from noise. The robust signal is the statistical signature: on a near-uniform model top-p and top-k emit visibly different numbers of DISTINCT tokens (61 vs 257 in 300 tokens at vocab=1109), which is a property of the candidate set rather than of any rand() draw. Suite [47e] asserts a restrictive nucleus narrows the emitted set and that out-of-range top_p falls back to top-k without emptying the set. qsort comparator is NaN-safe: a NaN comparison yields 0 rather than an inconsistent ordering, which qsort is permitted to fault on. Suite: 3172/3172 on the http+model build. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 16 ++++++ src/model_infer.c | 71 ++++++++++++++++++++++----- tests/run_all_tests.sh | 15 ++++++ tests/test_top_p.sh | 107 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 13 deletions(-) create mode 100755 tests/test_top_p.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 70167e7c..ee5bb3b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,22 @@ All notable changes to EigenScript are documented here. being recomputed by render and click separately. ### Added +- **`eigen_generate` accepts an optional 4th argument: `top_p` (nucleus + sampling).** Keeps the smallest set of tokens whose cumulative probability + reaches `p`, so the candidate set adapts to the model's confidence, rather + than the fixed top-k=40 that admits 39 near-zero tokens where the + distribution is sharp and truncates real choices where it is flat. Absent or + outside `(0,1)` keeps the historical top-k path, so existing callers and + their recorded numbers are unchanged. Tested by statistical signature rather + than by comparing token sequences — generation draws from a global `rand()`, + so two calls with the *same* policy already disagree on most positions (3/40 + identical), and an earlier version of the check "verified" top-p from 0/40 + matches, which the control shows is meaningless. Suite `[47e]` asserts a + restrictive nucleus narrows the emitted candidate set. **Honest scope: this + did not fix the repetition it was written for.** The model assigns its + repeat-loop continuation p=0.987, so a 0.95 nucleus keeps exactly one token + and changes nothing — the degeneracy is a genuine fixed point in the model, + not a decoding artifact. - **`eigen_eval_loss` — forward-only held-out cross-entropy (model build).** Scores one held-out window without a backward pass, weight update, requantise, or observer write, so evaluating a checkpoint never mutates it. diff --git a/src/model_infer.c b/src/model_infer.c index 733f6b56..252f49db 100644 --- a/src/model_infer.c +++ b/src/model_infer.c @@ -424,7 +424,14 @@ static void native_forward(int *token_ids, int seq_len, TransformerModel *model, free(x); } -static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel *model, double temperature, int max_tokens, int *out_len) { +/* Descending order, NaN-safe: a NaN comparison yields 0 (treated equal) rather + * than an inconsistent ordering, which qsort is allowed to fault on. */ +static int cmp_prob_desc(const void *a, const void *b) { + float fa = *(const float *)a, fb = *(const float *)b; + return (fa < fb) - (fa > fb); +} + +static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel *model, double temperature, int max_tokens, double top_p, int *out_len) { /* temperature controls sampling: < 0.01 = greedy argmax, otherwise top-k sampling * Returns a caller-owned int array of generated token IDs. Caller must free. */ int vocab_size = model->config.vocab_size; @@ -481,19 +488,51 @@ static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel for (int i = 0; i < vocab_size; i++) logits[i] /= sum; } - int top_k = 40; - if (top_k > vocab_size) top_k = vocab_size; + /* Truncate the tail before sampling. Two policies: + * + * top-p (nucleus): keep the smallest set of tokens whose cumulative + * probability reaches p. The candidate set ADAPTS to how confident + * the model is -- narrow where the next token is nearly determined + * (after `is`, or mid-keyword), wide where it genuinely branches. + * + * top-k: keep a fixed 40 regardless. That is the failure mode this + * model actually hits: where the distribution is sharp, a fixed k + * still admits 39 low-probability tokens and renormalises them up, + * and where it is flat, k=40 truncates real choices. Generation + * collapsed onto the single most frequent identifier (`b is b + 0` + * repeated, 7 distinct identifiers in a whole sample) even at + * held-out perplexity 5.07, i.e. with the model fitting fine. + * + * top_p <= 0 or >= 1 keeps the historical top-k path, so existing + * callers and their recorded numbers are unchanged. */ float probs_sorted[VOCAB_SIZE]; - memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float)); - for (int j = 0; j < top_k; j++) { - int max_idx = j; - for (int i = j + 1; i < vocab_size; i++) - if (probs_sorted[i] > probs_sorted[max_idx]) max_idx = i; - float tmp = probs_sorted[j]; - probs_sorted[j] = probs_sorted[max_idx]; - probs_sorted[max_idx] = tmp; + float threshold; + if (top_p > 0.0 && top_p < 1.0) { + memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float)); + qsort(probs_sorted, (size_t)vocab_size, sizeof(float), cmp_prob_desc); + float cum = 0.0f; + threshold = probs_sorted[0]; + for (int i = 0; i < vocab_size; i++) { + cum += probs_sorted[i]; + if (cum >= (float)top_p) { threshold = probs_sorted[i]; break; } + } + /* Never empty the candidate set: if rounding leaves the cutoff + * above every probability, the argmax alone must survive. */ + if (threshold > probs_sorted[0]) threshold = probs_sorted[0]; + } else { + int top_k = 40; + if (top_k > vocab_size) top_k = vocab_size; + memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float)); + for (int j = 0; j < top_k; j++) { + int max_idx = j; + for (int i = j + 1; i < vocab_size; i++) + if (probs_sorted[i] > probs_sorted[max_idx]) max_idx = i; + float tmp = probs_sorted[j]; + probs_sorted[j] = probs_sorted[max_idx]; + probs_sorted[max_idx] = tmp; + } + threshold = probs_sorted[top_k - 1]; } - float threshold = probs_sorted[top_k - 1]; float filtered_sum = 0.0f; for (int i = 0; i < vocab_size; i++) { if (logits[i] < threshold) logits[i] = 0.0f; @@ -548,8 +587,14 @@ Value* builtin_eigen_generate(Value *arg) { double temperature = 0.1; int max_tokens = 80; + /* Optional 4th arg: top_p. Absent (or outside (0,1)) keeps the historical + * fixed top-k=40 path, so every existing caller and recorded number is + * unchanged -- decoding policy is opt-in, not a silent default flip. */ + double top_p = 0.0; if (arg->data.list.items[1]->type == VAL_NUM) temperature = arg->data.list.items[1]->data.num; if (arg->data.list.items[2]->type == VAL_NUM) max_tokens = (int)arg->data.list.items[2]->data.num; + if (arg->data.list.count >= 4 && arg->data.list.items[3]->type == VAL_NUM) + top_p = arg->data.list.items[3]->data.num; if (!g_model.loaded) return make_list(0); @@ -575,7 +620,7 @@ Value* builtin_eigen_generate(Value *arg) { } int out_len = 0; - int *output_ids = generate_response(prompt_ids, prompt_len, &g_model, temperature, max_tokens, &out_len); + int *output_ids = generate_response(prompt_ids, prompt_len, &g_model, temperature, max_tokens, top_p, &out_len); free(prompt_ids); Value *result = make_list(out_len); diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 0aa8a079..09aadc3f 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -1905,6 +1905,21 @@ if ! echo "$MODEL_PROBE_OUT" | grep -q "undefined variable"; then echo " PASS: untrained cross-entropy sits at ln(vocab)" fi echo "" + + echo "[47e/47] eigen_generate top-p nucleus sampling (4 checks)" + TP_OUTPUT=$(bash "$TESTS_DIR/test_top_p.sh" 2>&1) + TP_PASS=$(echo "$TP_OUTPUT" | grep -c "PASS:" || true) + TP_FAIL=$(echo "$TP_OUTPUT" | grep -c "FAIL:" || true) + TOTAL=$((TOTAL + TP_PASS + TP_FAIL)) + PASS=$((PASS + TP_PASS)) + FAIL=$((FAIL + TP_FAIL)) + if [ "$TP_FAIL" -gt 0 ]; then + echo " FAIL: $TP_FAIL top-p check(s) failed" + echo "$TP_OUTPUT" | grep "FAIL:" | head -4 + else + echo " PASS: top-p narrows the candidate set; out-of-range falls back to top-k" + fi + echo "" else echo "[47/47] Model roundtrip SKIPPED (binary built without EIGENSCRIPT_EXT_MODEL)" echo "" diff --git a/tests/test_top_p.sh b/tests/test_top_p.sh new file mode 100755 index 00000000..68888739 --- /dev/null +++ b/tests/test_top_p.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# eigen_generate top-p (nucleus) sampling. +# +# Testing this needs care: generation draws from a global rand(), so two calls +# with the SAME policy already disagree on most positions (measured 3/40 +# identical). Comparing two token sequences therefore proves nothing about the +# sampling policy -- an earlier version of this check "verified" top-p by +# observing 0/40 matches, which is indistinguishable from the same-policy +# control. +# +# What IS robust is the statistical signature. On a near-uniform (untrained) +# model, top-p keeps every token up to a cumulative mass of p while top-k keeps +# a fixed few, so the two policies emit visibly different numbers of DISTINCT +# tokens over a long generation. That is a property of the candidate set, not +# of any particular rand() draw. On the vocab=1109 model the split is 61 vs 257 +# distinct in 300 tokens. +# +# Runs only when the binary is built with EIGENSCRIPT_EXT_MODEL=1. + +set -u +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +EIGS="$TESTS_DIR/../src/eigenscript" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +fail() { echo " FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); } + +MODEL=/tmp/eigs_tp_tiny.json +HARNESS=/tmp/eigs_tp_harness.eigs +cleanup() { rm -f "$MODEL" "$HARNESS" /tmp/eigs_tp_*.log; } +trap cleanup EXIT + +if ! "$EIGS" "$TESTS_DIR/gen_tiny_model.eigs" > "$MODEL" 2>/tmp/eigs_tp_gen.log; then + fail "TP00 generate tiny model" "see /tmp/eigs_tp_gen.log" + echo "TOPP: 0 passed, 1 failed" + exit 1 +fi + +# vocab=8, so the top-k=40 default clamps to "keep everything" and a restrictive +# top-p must keep strictly fewer. 400 tokens makes the distinct counts stable. +cat > "$HARNESS" <<'EIGS' +ck is args of null +r is eigen_model_load of ck[0] +define ndistinct(ids) as: + seen is zeros of 64 + d is 0 + for i in range of (len of ids): + t is ids[i] + if t >= 0: + if t < 64: + if seen[t] == 0: + seen[t] is 1 + d is d + 1 + return d +a is eigen_generate of [[1, 2, 3], 1.0, 400] +b is eigen_generate of [[1, 2, 3], 1.0, 400, 0.3] +print of ("K " + (str of (ndistinct of a))) +print of ("P " + (str of (ndistinct of b))) +print of ("PLEN " + (str of (len of b))) +EIGS + +OUT=$("$EIGS" "$HARNESS" "$MODEL" 2>/dev/null) +K=$(printf '%s\n' "$OUT" | sed -n 's/^K //p') +P=$(printf '%s\n' "$OUT" | sed -n 's/^P //p') +PLEN=$(printf '%s\n' "$OUT" | sed -n 's/^PLEN //p') + +if [ -z "$K" ] || [ -z "$P" ]; then + fail "TP01 both sampling policies produced output" "K='$K' P='$P'" + echo "TOPP: $PASS passed, $((FAIL)) failed" + exit 1 +fi +ok "TP01 both sampling policies produced output" + +if [ "$PLEN" = "400" ]; then + ok "TP02 top-p generated the requested token count" +else + fail "TP02 top-p token count" "expected 400, got $PLEN" +fi + +# The load-bearing assertion: a restrictive nucleus must narrow the candidate +# set. If top_p were ignored, this collapses to the same policy twice and the +# counts land equal. +if [ "$P" -lt "$K" ]; then + ok "TP03 top-p=0.3 narrows the candidate set vs top-k ($P < $K distinct)" +else + fail "TP03 top-p did not narrow the candidate set" "top-k=$K distinct, top-p=$P distinct" +fi + +# top_p outside (0,1) must fall back to the historical top-k path rather than +# emptying the candidate set or dividing by zero. +cat > "$HARNESS" <<'EIGS' +ck is args of null +r is eigen_model_load of ck[0] +for p in [0.0, 1.0, 2.0, 0 - 1.0]: + g is eigen_generate of [[1, 2, 3], 1.0, 20, p] + print of ("N " + (str of (len of g))) +EIGS +OOR=$("$EIGS" "$HARNESS" "$MODEL" 2>/dev/null | sed -n 's/^N //p' | sort -u) +if [ "$OOR" = "20" ]; then + ok "TP04 out-of-range top_p falls back to top-k and still generates" +else + fail "TP04 out-of-range top_p broke generation" "lengths: $(printf '%s ' $OOR)" +fi + +echo "TOPP: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]