Add eigen_eval_loss: forward-only held-out cross-entropy

CHANGELOG.md

@@ -69,6 +69,38 @@ All notable changes to EigenScript are documented here.
   button geometry is now defined once (`_dialog_btn_rect`) instead of
   being recomputed by render and click separately.
 
+### Added
+- **`eigen_generate` accepts an optional 4th argument: `top_p` (nucleus
+  sampling).** Keeps the smallest set of tokens whose cumulative probability
+  reaches `p`, so the candidate set adapts to the model's confidence, rather
+  than the fixed top-k=40 that admits 39 near-zero tokens where the
+  distribution is sharp and truncates real choices where it is flat. Absent or
+  outside `(0,1)` keeps the historical top-k path, so existing callers and
+  their recorded numbers are unchanged. Tested by statistical signature rather
+  than by comparing token sequences — generation draws from a global `rand()`,
+  so two calls with the *same* policy already disagree on most positions (3/40
+  identical), and an earlier version of the check "verified" top-p from 0/40
+  matches, which the control shows is meaningless. Suite `[47e]` asserts a
+  restrictive nucleus narrows the emitted candidate set. **Honest scope: this
+  did not fix the repetition it was written for.** The model assigns its
+  repeat-loop continuation p=0.987, so a 0.95 nucleus keeps exactly one token
+  and changes nothing — the degeneracy is a genuine fixed point in the model,
+  not a decoding artifact.
+- **`eigen_eval_loss` — forward-only held-out cross-entropy (model build).**
+  Scores one held-out window without a backward pass, weight update,
+  requantise, or observer write, so evaluating a checkpoint never mutates it.
+  This is the metric a language model should be *selected* on, and it did not
+  previously exist: iLambdaAi ranked checkpoints on a generated-and-parsed
+  rate at n=30, where a 20% score has a 95% interval roughly 8%-39%. That
+  metric could not distinguish two models whose held-out perplexity differed
+  by 7x (39.6 vs 5.76) — it scored both at an identical 60/300, because greedy
+  decoding collapses onto frequency attractors regardless of model quality.
+  A loss gives one datapoint per window on a continuous scale instead of 300
+  binary trials. Pinned by suite `[47d]` against the property that an
+  untrained model must score ln(vocab_size) — measured 7.004 against a
+  predicted 7.011 at vocab=1109 — which simultaneously catches a missing
+  max-subtraction (overflow to inf), a mis-indexed target, and sign errors.
+
 ### Changed
 - **The observer learning-rate gate is now two-channel (entropy *and* drift).**
   `observer_matrix_scale` throttled a weight matrix to 0.5x whenever its

src/ext_names.h

@@ -110,6 +110,7 @@
 #define EIGS_MODEL_BUILTINS(X) \
     X(eigen_model_loaded, builtin_eigen_model_loaded) \
     X(eigen_generate, builtin_eigen_generate) \
+    X(eigen_eval_loss, builtin_eigen_eval_loss) \
     X(eigen_model_info, builtin_eigen_model_info) \
     X(native_train_step_builtin, builtin_native_train_step) \
     X(model_save_weights, builtin_model_save_weights) \

src/model_infer.c

@@ -424,7 +424,14 @@ static void native_forward(int *token_ids, int seq_len, TransformerModel *model,
     free(x);
 }
 
-static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel *model, double temperature, int max_tokens, int *out_len) {
+/* Descending order, NaN-safe: a NaN comparison yields 0 (treated equal) rather
+ * than an inconsistent ordering, which qsort is allowed to fault on. */
+static int cmp_prob_desc(const void *a, const void *b) {
+    float fa = *(const float *)a, fb = *(const float *)b;
+    return (fa < fb) - (fa > fb);
+}
+
+static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel *model, double temperature, int max_tokens, double top_p, int *out_len) {
     /* temperature controls sampling: < 0.01 = greedy argmax, otherwise top-k sampling
      * Returns a caller-owned int array of generated token IDs. Caller must free. */
     int vocab_size = model->config.vocab_size;
@@ -481,19 +488,51 @@ static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel
                 for (int i = 0; i < vocab_size; i++) logits[i] /= sum;
             }
 
-            int top_k = 40;
-            if (top_k > vocab_size) top_k = vocab_size;
+            /* Truncate the tail before sampling. Two policies:
+             *
+             * top-p (nucleus): keep the smallest set of tokens whose cumulative
+             * probability reaches p. The candidate set ADAPTS to how confident
+             * the model is -- narrow where the next token is nearly determined
+             * (after `is`, or mid-keyword), wide where it genuinely branches.
+             *
+             * top-k: keep a fixed 40 regardless. That is the failure mode this
+             * model actually hits: where the distribution is sharp, a fixed k
+             * still admits 39 low-probability tokens and renormalises them up,
+             * and where it is flat, k=40 truncates real choices. Generation
+             * collapsed onto the single most frequent identifier (`b is b + 0`
+             * repeated, 7 distinct identifiers in a whole sample) even at
+             * held-out perplexity 5.07, i.e. with the model fitting fine.
+             *
+             * top_p <= 0 or >= 1 keeps the historical top-k path, so existing
+             * callers and their recorded numbers are unchanged. */
             float probs_sorted[VOCAB_SIZE];
-            memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float));
-            for (int j = 0; j < top_k; j++) {
-                int max_idx = j;
-                for (int i = j + 1; i < vocab_size; i++)
-                    if (probs_sorted[i] > probs_sorted[max_idx]) max_idx = i;
-                float tmp = probs_sorted[j];
-                probs_sorted[j] = probs_sorted[max_idx];
-                probs_sorted[max_idx] = tmp;
+            float threshold;
+            if (top_p > 0.0 && top_p < 1.0) {
+                memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float));
+                qsort(probs_sorted, (size_t)vocab_size, sizeof(float), cmp_prob_desc);
+                float cum = 0.0f;
+                threshold = probs_sorted[0];
+                for (int i = 0; i < vocab_size; i++) {
+                    cum += probs_sorted[i];
+                    if (cum >= (float)top_p) { threshold = probs_sorted[i]; break; }
+                }
+                /* Never empty the candidate set: if rounding leaves the cutoff
+                 * above every probability, the argmax alone must survive. */
+                if (threshold > probs_sorted[0]) threshold = probs_sorted[0];
+            } else {
+                int top_k = 40;
+                if (top_k > vocab_size) top_k = vocab_size;
+                memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float));
+                for (int j = 0; j < top_k; j++) {
+                    int max_idx = j;
+                    for (int i = j + 1; i < vocab_size; i++)
+                        if (probs_sorted[i] > probs_sorted[max_idx]) max_idx = i;
+                    float tmp = probs_sorted[j];
+                    probs_sorted[j] = probs_sorted[max_idx];
+                    probs_sorted[max_idx] = tmp;
+                }
+                threshold = probs_sorted[top_k - 1];
             }
-            float threshold = probs_sorted[top_k - 1];
             float filtered_sum = 0.0f;
             for (int i = 0; i < vocab_size; i++) {
                 if (logits[i] < threshold) logits[i] = 0.0f;
@@ -548,8 +587,14 @@ Value* builtin_eigen_generate(Value *arg) {
 
     double temperature = 0.1;
     int max_tokens = 80;
+    /* Optional 4th arg: top_p. Absent (or outside (0,1)) keeps the historical
+     * fixed top-k=40 path, so every existing caller and recorded number is
+     * unchanged -- decoding policy is opt-in, not a silent default flip. */
+    double top_p = 0.0;
     if (arg->data.list.items[1]->type == VAL_NUM) temperature = arg->data.list.items[1]->data.num;
     if (arg->data.list.items[2]->type == VAL_NUM) max_tokens = (int)arg->data.list.items[2]->data.num;
+    if (arg->data.list.count >= 4 && arg->data.list.items[3]->type == VAL_NUM)
+        top_p = arg->data.list.items[3]->data.num;
 
     if (!g_model.loaded) return make_list(0);
 
@@ -575,7 +620,7 @@ Value* builtin_eigen_generate(Value *arg) {
     }
 
     int out_len = 0;
-    int *output_ids = generate_response(prompt_ids, prompt_len, &g_model, temperature, max_tokens, &out_len);
+    int *output_ids = generate_response(prompt_ids, prompt_len, &g_model, temperature, max_tokens, top_p, &out_len);
     free(prompt_ids);
 
     Value *result = make_list(out_len);
@@ -586,6 +631,76 @@ Value* builtin_eigen_generate(Value *arg) {
     return result;
 }
 
+Value* builtin_eigen_eval_loss(Value *arg) {
+    /* Input: [prompt_ids_list, target_id]
+     * Output: cross-entropy of target_id given the prompt, in nats.
+     *
+     * Held-out loss is the selection metric a language model should be ranked
+     * on, and this is the forward-only path that makes it computable. The task
+     * metrics (parse rate) sample 300 generations; this scores one point per
+     * WINDOW, so a few thousand windows give ~10x the samples on a continuous
+     * scale instead of a binary one. That difference is not academic: a night
+     * of A/B runs on the n=30 parse rate could not distinguish two arms whose
+     * accuracy differed by 14 points, because a pass/fail metric at n=30 has a
+     * 95% interval about 30 points wide.
+     *
+     * Forward only -- no backward, no weight update, no requantise, no observer
+     * state, and g_model_age/g_training_samples are untouched, so scoring a
+     * checkpoint never mutates it. */
+    if (!arg || arg->type != VAL_LIST || arg->data.list.count < 2) {
+        fprintf(stderr, "eigen_eval_loss: requires [prompt_ids, target_id]\n");
+        return make_num(-1.0);
+    }
+    Value *prompt_list = arg->data.list.items[0];
+    if (prompt_list->type != VAL_LIST) {
+        fprintf(stderr, "eigen_eval_loss: prompt_ids must be a list\n");
+        return make_num(-1.0);
+    }
+    if (arg->data.list.items[1]->type != VAL_NUM) {
+        fprintf(stderr, "eigen_eval_loss: target_id must be a number\n");
+        return make_num(-1.0);
+    }
+    int target_id = (int)arg->data.list.items[1]->data.num;
+
+    if (!g_model.loaded) return make_num(-1.0);
+    int vocab_size = g_model.config.vocab_size;
+    if (target_id < 0 || target_id >= vocab_size) {
+        fprintf(stderr, "eigen_eval_loss: target_id %d out of range [0,%d)\n", target_id, vocab_size);
+        return make_num(-1.0);
+    }
+
+    int prompt_len = prompt_list->data.list.count;
+    if (prompt_len <= 0) {
+        fprintf(stderr, "eigen_eval_loss: prompt must be non-empty\n");
+        return make_num(-1.0);
+    }
+
+    int *prompt_ids = xcalloc(prompt_len, sizeof(int));
+    for (int i = 0; i < prompt_len; i++) {
+        Value *v = prompt_list->data.list.items[i];
+        int t = (v->type == VAL_NUM) ? (int)v->data.num : 0;
+        if (t < 0 || t >= vocab_size) t = 0;
+        prompt_ids[i] = t;
+    }
+
+    float *logits = xcalloc(vocab_size, sizeof(float));
+    native_forward(prompt_ids, prompt_len, &g_model, logits);
+    free(prompt_ids);
+
+    /* Numerically stable log-softmax at the target: subtract the max before
+     * exponentiating, or a large logit overflows float and returns inf/nan --
+     * which would silently poison the mean over thousands of windows. */
+    float maxl = logits[0];
+    for (int j = 1; j < vocab_size; j++) if (logits[j] > maxl) maxl = logits[j];
+    double sum = 0.0;
+    for (int j = 0; j < vocab_size; j++) sum += exp((double)(logits[j] - maxl));
+    double loss = -((double)(logits[target_id] - maxl) - log(sum));
+    free(logits);
+
+    if (isnan(loss) || isinf(loss)) return make_num(-1.0);
+    return make_num(loss);
+}
+
 Value* builtin_eigen_model_info(Value *arg) {
     (void)arg;
     char buf[512];

src/model_internal.h

@@ -220,6 +220,7 @@ Value* builtin_eigen_model_save_binary(Value *arg);
 Value* builtin_eigen_checkpoint_info(Value *arg);
 Value* builtin_eigen_generate(Value *arg);
 Value* builtin_eigen_model_loaded(Value *arg);
+Value* builtin_eigen_eval_loss(Value *arg);
 Value* builtin_eigen_model_info(Value *arg);
 Value* builtin_native_train_step(Value *arg);
 

tests/run_all_tests.sh

@@ -1890,6 +1890,36 @@ if ! echo "$MODEL_PROBE_OUT" | grep -q "undefined variable"; then
         echo "  PASS: batched training path is gradient-identical to the per-position oracle"
     fi
     echo ""
+
+    echo "[47d/47] eigen_eval_loss held-out cross-entropy (4 checks)"
+    EL_OUTPUT=$(bash "$TESTS_DIR/test_eval_loss.sh" 2>&1)
+    EL_PASS=$(echo "$EL_OUTPUT" | grep -c "PASS:" || true)
+    EL_FAIL=$(echo "$EL_OUTPUT" | grep -c "FAIL:" || true)
+    TOTAL=$((TOTAL + EL_PASS + EL_FAIL))
+    PASS=$((PASS + EL_PASS))
+    FAIL=$((FAIL + EL_FAIL))
+    if [ "$EL_FAIL" -gt 0 ]; then
+        echo "  FAIL: $EL_FAIL eigen_eval_loss check(s) failed"
+        echo "$EL_OUTPUT" | grep "FAIL:" | head -4
+    else
+        echo "  PASS: untrained cross-entropy sits at ln(vocab)"
+    fi
+    echo ""
+
+    echo "[47e/47] eigen_generate top-p nucleus sampling (4 checks)"
+    TP_OUTPUT=$(bash "$TESTS_DIR/test_top_p.sh" 2>&1)
+    TP_PASS=$(echo "$TP_OUTPUT" | grep -c "PASS:" || true)
+    TP_FAIL=$(echo "$TP_OUTPUT" | grep -c "FAIL:" || true)
+    TOTAL=$((TOTAL + TP_PASS + TP_FAIL))
+    PASS=$((PASS + TP_PASS))
+    FAIL=$((FAIL + TP_FAIL))
+    if [ "$TP_FAIL" -gt 0 ]; then
+        echo "  FAIL: $TP_FAIL top-p check(s) failed"
+        echo "$TP_OUTPUT" | grep "FAIL:" | head -4
+    else
+        echo "  PASS: top-p narrows the candidate set; out-of-range falls back to top-k"
+    fi
+    echo ""
 else
     echo "[47/47] Model roundtrip SKIPPED (binary built without EIGENSCRIPT_EXT_MODEL)"
     echo ""

tests/test_eval_loss.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# eigen_eval_loss: forward-only held-out cross-entropy.
+#
+# The property that pins this down without needing a trained model: an
+# UNTRAINED model's output distribution is near-uniform over the vocabulary,
+# so its cross-entropy must sit at ln(vocab_size) whatever the target is. The
+# tiny fixture has vocab=8, so every loss must land near ln(8) = 2.0794.
+#
+# That single anchor catches the realistic failure modes at once -- a missing
+# max-subtraction (overflow -> inf), summing the softmax over the wrong axis,
+# indexing the wrong target, or returning logits instead of a loss would all
+# miss ln(8). It also catches sign errors, since a cross-entropy is never
+# negative.
+#
+# Runs only when the binary is built with EIGENSCRIPT_EXT_MODEL=1.
+
+set -u
+TESTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+EIGS="$TESTS_DIR/../src/eigenscript"
+
+PASS=0
+FAIL=0
+ok()   { echo "  PASS: $1"; PASS=$((PASS+1)); }
+fail() { echo "  FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); }
+
+MODEL=/tmp/eigs_el_tiny.json
+HARNESS=/tmp/eigs_el_harness.eigs
+cleanup() { rm -f "$MODEL" "$HARNESS" /tmp/eigs_el_*.log; }
+trap cleanup EXIT
+
+if ! "$EIGS" "$TESTS_DIR/gen_tiny_model.eigs" > "$MODEL" 2>/tmp/eigs_el_gen.log; then
+    fail "EL00 generate tiny model" "see /tmp/eigs_el_gen.log"
+    echo "EVALLOSS: 0 passed, 1 failed"
+    exit 1
+fi
+
+cat > "$HARNESS" <<EIGS
+r is eigen_model_load of "$MODEL"
+prompt is [1, 2, 3, 4]
+# Every vocab target on an untrained model: all must be near ln(8).
+for t in range of 8:
+    l is eigen_eval_loss of [prompt, t]
+    print of ("L " + (str of l))
+# Out-of-range target must be rejected with the -1 sentinel, not a garbage
+# read past the logits array.
+print of ("OOR " + (str of (eigen_eval_loss of [prompt, 999])))
+print of ("NEG " + (str of (eigen_eval_loss of [prompt, 0 - 1])))
+EIGS
+
+OUT=$("$EIGS" "$HARNESS" 2>/dev/null)
+LOSSES=$(printf '%s\n' "$OUT" | sed -n 's/^L //p')
+
+if [ -z "$LOSSES" ]; then
+    fail "EL01 eigen_eval_loss returned values" "no output"
+    echo "EVALLOSS: $PASS passed, $((FAIL+1)) failed"
+    exit 1
+fi
+ok "EL01 eigen_eval_loss returned a loss for every target"
+
+# All 8 losses within 0.35 nats of ln(8). The band is wide enough for the
+# fixture's small random weights to tilt the distribution, tight enough that
+# any structural error lands far outside it.
+BAD=$(printf '%s\n' "$LOSSES" | awk '
+    BEGIN { ln8 = 2.0794415; bad = 0 }
+    { d = $1 - ln8; if (d < 0) d = -d; if (d > 0.35) bad++ }
+    END { print bad }')
+if [ "$BAD" = "0" ]; then
+    ok "EL02 untrained losses sit at ln(vocab) (all 8 within 0.35 of 2.0794)"
+else
+    fail "EL02 untrained losses deviate from ln(vocab)" "$BAD of 8 outside band"
+    echo "  losses: $(printf '%s ' $LOSSES)"
+fi
+
+NONNEG=$(printf '%s\n' "$LOSSES" | awk '{ if ($1 < 0) n++ } END { print n+0 }')
+if [ "$NONNEG" = "0" ]; then
+    ok "EL03 no negative cross-entropy"
+else
+    fail "EL03 negative cross-entropy returned" "$NONNEG value(s) < 0"
+fi
+
+OOR=$(printf '%s\n' "$OUT" | sed -n 's/^OOR //p')
+NEG=$(printf '%s\n' "$OUT" | sed -n 's/^NEG //p')
+if [ "${OOR%%.*}" = "-1" ] && [ "${NEG%%.*}" = "-1" ]; then
+    ok "EL04 out-of-range targets rejected with the -1 sentinel"
+else
+    fail "EL04 out-of-range target not rejected" "target=999 -> '$OOR', target=-1 -> '$NEG'"
+fi
+
+echo "EVALLOSS: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ]