diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ebb482..ee5bb3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,38 @@ All notable changes to EigenScript are documented here. button geometry is now defined once (`_dialog_btn_rect`) instead of being recomputed by render and click separately. +### Added +- **`eigen_generate` accepts an optional 4th argument: `top_p` (nucleus + sampling).** Keeps the smallest set of tokens whose cumulative probability + reaches `p`, so the candidate set adapts to the model's confidence, rather + than the fixed top-k=40 that admits 39 near-zero tokens where the + distribution is sharp and truncates real choices where it is flat. Absent or + outside `(0,1)` keeps the historical top-k path, so existing callers and + their recorded numbers are unchanged. Tested by statistical signature rather + than by comparing token sequences — generation draws from a global `rand()`, + so two calls with the *same* policy already disagree on most positions (3/40 + identical), and an earlier version of the check "verified" top-p from 0/40 + matches, which the control shows is meaningless. Suite `[47e]` asserts a + restrictive nucleus narrows the emitted candidate set. **Honest scope: this + did not fix the repetition it was written for.** The model assigns its + repeat-loop continuation p=0.987, so a 0.95 nucleus keeps exactly one token + and changes nothing — the degeneracy is a genuine fixed point in the model, + not a decoding artifact. +- **`eigen_eval_loss` — forward-only held-out cross-entropy (model build).** + Scores one held-out window without a backward pass, weight update, + requantise, or observer write, so evaluating a checkpoint never mutates it. + This is the metric a language model should be *selected* on, and it did not + previously exist: iLambdaAi ranked checkpoints on a generated-and-parsed + rate at n=30, where a 20% score has a 95% interval roughly 8%-39%. That + metric could not distinguish two models whose held-out perplexity differed + by 7x (39.6 vs 5.76) — it scored both at an identical 60/300, because greedy + decoding collapses onto frequency attractors regardless of model quality. + A loss gives one datapoint per window on a continuous scale instead of 300 + binary trials. Pinned by suite `[47d]` against the property that an + untrained model must score ln(vocab_size) — measured 7.004 against a + predicted 7.011 at vocab=1109 — which simultaneously catches a missing + max-subtraction (overflow to inf), a mis-indexed target, and sign errors. + ### Changed - **The observer learning-rate gate is now two-channel (entropy *and* drift).** `observer_matrix_scale` throttled a weight matrix to 0.5x whenever its diff --git a/src/ext_names.h b/src/ext_names.h index c90517d..148a587 100644 --- a/src/ext_names.h +++ b/src/ext_names.h @@ -110,6 +110,7 @@ #define EIGS_MODEL_BUILTINS(X) \ X(eigen_model_loaded, builtin_eigen_model_loaded) \ X(eigen_generate, builtin_eigen_generate) \ + X(eigen_eval_loss, builtin_eigen_eval_loss) \ X(eigen_model_info, builtin_eigen_model_info) \ X(native_train_step_builtin, builtin_native_train_step) \ X(model_save_weights, builtin_model_save_weights) \ diff --git a/src/model_infer.c b/src/model_infer.c index 1a9acb9..252f49d 100644 --- a/src/model_infer.c +++ b/src/model_infer.c @@ -424,7 +424,14 @@ static void native_forward(int *token_ids, int seq_len, TransformerModel *model, free(x); } -static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel *model, double temperature, int max_tokens, int *out_len) { +/* Descending order, NaN-safe: a NaN comparison yields 0 (treated equal) rather + * than an inconsistent ordering, which qsort is allowed to fault on. */ +static int cmp_prob_desc(const void *a, const void *b) { + float fa = *(const float *)a, fb = *(const float *)b; + return (fa < fb) - (fa > fb); +} + +static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel *model, double temperature, int max_tokens, double top_p, int *out_len) { /* temperature controls sampling: < 0.01 = greedy argmax, otherwise top-k sampling * Returns a caller-owned int array of generated token IDs. Caller must free. */ int vocab_size = model->config.vocab_size; @@ -481,19 +488,51 @@ static int* generate_response(int *prompt_ids, int prompt_len, TransformerModel for (int i = 0; i < vocab_size; i++) logits[i] /= sum; } - int top_k = 40; - if (top_k > vocab_size) top_k = vocab_size; + /* Truncate the tail before sampling. Two policies: + * + * top-p (nucleus): keep the smallest set of tokens whose cumulative + * probability reaches p. The candidate set ADAPTS to how confident + * the model is -- narrow where the next token is nearly determined + * (after `is`, or mid-keyword), wide where it genuinely branches. + * + * top-k: keep a fixed 40 regardless. That is the failure mode this + * model actually hits: where the distribution is sharp, a fixed k + * still admits 39 low-probability tokens and renormalises them up, + * and where it is flat, k=40 truncates real choices. Generation + * collapsed onto the single most frequent identifier (`b is b + 0` + * repeated, 7 distinct identifiers in a whole sample) even at + * held-out perplexity 5.07, i.e. with the model fitting fine. + * + * top_p <= 0 or >= 1 keeps the historical top-k path, so existing + * callers and their recorded numbers are unchanged. */ float probs_sorted[VOCAB_SIZE]; - memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float)); - for (int j = 0; j < top_k; j++) { - int max_idx = j; - for (int i = j + 1; i < vocab_size; i++) - if (probs_sorted[i] > probs_sorted[max_idx]) max_idx = i; - float tmp = probs_sorted[j]; - probs_sorted[j] = probs_sorted[max_idx]; - probs_sorted[max_idx] = tmp; + float threshold; + if (top_p > 0.0 && top_p < 1.0) { + memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float)); + qsort(probs_sorted, (size_t)vocab_size, sizeof(float), cmp_prob_desc); + float cum = 0.0f; + threshold = probs_sorted[0]; + for (int i = 0; i < vocab_size; i++) { + cum += probs_sorted[i]; + if (cum >= (float)top_p) { threshold = probs_sorted[i]; break; } + } + /* Never empty the candidate set: if rounding leaves the cutoff + * above every probability, the argmax alone must survive. */ + if (threshold > probs_sorted[0]) threshold = probs_sorted[0]; + } else { + int top_k = 40; + if (top_k > vocab_size) top_k = vocab_size; + memcpy(probs_sorted, logits, (size_t)vocab_size * sizeof(float)); + for (int j = 0; j < top_k; j++) { + int max_idx = j; + for (int i = j + 1; i < vocab_size; i++) + if (probs_sorted[i] > probs_sorted[max_idx]) max_idx = i; + float tmp = probs_sorted[j]; + probs_sorted[j] = probs_sorted[max_idx]; + probs_sorted[max_idx] = tmp; + } + threshold = probs_sorted[top_k - 1]; } - float threshold = probs_sorted[top_k - 1]; float filtered_sum = 0.0f; for (int i = 0; i < vocab_size; i++) { if (logits[i] < threshold) logits[i] = 0.0f; @@ -548,8 +587,14 @@ Value* builtin_eigen_generate(Value *arg) { double temperature = 0.1; int max_tokens = 80; + /* Optional 4th arg: top_p. Absent (or outside (0,1)) keeps the historical + * fixed top-k=40 path, so every existing caller and recorded number is + * unchanged -- decoding policy is opt-in, not a silent default flip. */ + double top_p = 0.0; if (arg->data.list.items[1]->type == VAL_NUM) temperature = arg->data.list.items[1]->data.num; if (arg->data.list.items[2]->type == VAL_NUM) max_tokens = (int)arg->data.list.items[2]->data.num; + if (arg->data.list.count >= 4 && arg->data.list.items[3]->type == VAL_NUM) + top_p = arg->data.list.items[3]->data.num; if (!g_model.loaded) return make_list(0); @@ -575,7 +620,7 @@ Value* builtin_eigen_generate(Value *arg) { } int out_len = 0; - int *output_ids = generate_response(prompt_ids, prompt_len, &g_model, temperature, max_tokens, &out_len); + int *output_ids = generate_response(prompt_ids, prompt_len, &g_model, temperature, max_tokens, top_p, &out_len); free(prompt_ids); Value *result = make_list(out_len); @@ -586,6 +631,76 @@ Value* builtin_eigen_generate(Value *arg) { return result; } +Value* builtin_eigen_eval_loss(Value *arg) { + /* Input: [prompt_ids_list, target_id] + * Output: cross-entropy of target_id given the prompt, in nats. + * + * Held-out loss is the selection metric a language model should be ranked + * on, and this is the forward-only path that makes it computable. The task + * metrics (parse rate) sample 300 generations; this scores one point per + * WINDOW, so a few thousand windows give ~10x the samples on a continuous + * scale instead of a binary one. That difference is not academic: a night + * of A/B runs on the n=30 parse rate could not distinguish two arms whose + * accuracy differed by 14 points, because a pass/fail metric at n=30 has a + * 95% interval about 30 points wide. + * + * Forward only -- no backward, no weight update, no requantise, no observer + * state, and g_model_age/g_training_samples are untouched, so scoring a + * checkpoint never mutates it. */ + if (!arg || arg->type != VAL_LIST || arg->data.list.count < 2) { + fprintf(stderr, "eigen_eval_loss: requires [prompt_ids, target_id]\n"); + return make_num(-1.0); + } + Value *prompt_list = arg->data.list.items[0]; + if (prompt_list->type != VAL_LIST) { + fprintf(stderr, "eigen_eval_loss: prompt_ids must be a list\n"); + return make_num(-1.0); + } + if (arg->data.list.items[1]->type != VAL_NUM) { + fprintf(stderr, "eigen_eval_loss: target_id must be a number\n"); + return make_num(-1.0); + } + int target_id = (int)arg->data.list.items[1]->data.num; + + if (!g_model.loaded) return make_num(-1.0); + int vocab_size = g_model.config.vocab_size; + if (target_id < 0 || target_id >= vocab_size) { + fprintf(stderr, "eigen_eval_loss: target_id %d out of range [0,%d)\n", target_id, vocab_size); + return make_num(-1.0); + } + + int prompt_len = prompt_list->data.list.count; + if (prompt_len <= 0) { + fprintf(stderr, "eigen_eval_loss: prompt must be non-empty\n"); + return make_num(-1.0); + } + + int *prompt_ids = xcalloc(prompt_len, sizeof(int)); + for (int i = 0; i < prompt_len; i++) { + Value *v = prompt_list->data.list.items[i]; + int t = (v->type == VAL_NUM) ? (int)v->data.num : 0; + if (t < 0 || t >= vocab_size) t = 0; + prompt_ids[i] = t; + } + + float *logits = xcalloc(vocab_size, sizeof(float)); + native_forward(prompt_ids, prompt_len, &g_model, logits); + free(prompt_ids); + + /* Numerically stable log-softmax at the target: subtract the max before + * exponentiating, or a large logit overflows float and returns inf/nan -- + * which would silently poison the mean over thousands of windows. */ + float maxl = logits[0]; + for (int j = 1; j < vocab_size; j++) if (logits[j] > maxl) maxl = logits[j]; + double sum = 0.0; + for (int j = 0; j < vocab_size; j++) sum += exp((double)(logits[j] - maxl)); + double loss = -((double)(logits[target_id] - maxl) - log(sum)); + free(logits); + + if (isnan(loss) || isinf(loss)) return make_num(-1.0); + return make_num(loss); +} + Value* builtin_eigen_model_info(Value *arg) { (void)arg; char buf[512]; diff --git a/src/model_internal.h b/src/model_internal.h index aef4b58..c55baaf 100644 --- a/src/model_internal.h +++ b/src/model_internal.h @@ -220,6 +220,7 @@ Value* builtin_eigen_model_save_binary(Value *arg); Value* builtin_eigen_checkpoint_info(Value *arg); Value* builtin_eigen_generate(Value *arg); Value* builtin_eigen_model_loaded(Value *arg); +Value* builtin_eigen_eval_loss(Value *arg); Value* builtin_eigen_model_info(Value *arg); Value* builtin_native_train_step(Value *arg); diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index da1eab7..09aadc3 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -1890,6 +1890,36 @@ if ! echo "$MODEL_PROBE_OUT" | grep -q "undefined variable"; then echo " PASS: batched training path is gradient-identical to the per-position oracle" fi echo "" + + echo "[47d/47] eigen_eval_loss held-out cross-entropy (4 checks)" + EL_OUTPUT=$(bash "$TESTS_DIR/test_eval_loss.sh" 2>&1) + EL_PASS=$(echo "$EL_OUTPUT" | grep -c "PASS:" || true) + EL_FAIL=$(echo "$EL_OUTPUT" | grep -c "FAIL:" || true) + TOTAL=$((TOTAL + EL_PASS + EL_FAIL)) + PASS=$((PASS + EL_PASS)) + FAIL=$((FAIL + EL_FAIL)) + if [ "$EL_FAIL" -gt 0 ]; then + echo " FAIL: $EL_FAIL eigen_eval_loss check(s) failed" + echo "$EL_OUTPUT" | grep "FAIL:" | head -4 + else + echo " PASS: untrained cross-entropy sits at ln(vocab)" + fi + echo "" + + echo "[47e/47] eigen_generate top-p nucleus sampling (4 checks)" + TP_OUTPUT=$(bash "$TESTS_DIR/test_top_p.sh" 2>&1) + TP_PASS=$(echo "$TP_OUTPUT" | grep -c "PASS:" || true) + TP_FAIL=$(echo "$TP_OUTPUT" | grep -c "FAIL:" || true) + TOTAL=$((TOTAL + TP_PASS + TP_FAIL)) + PASS=$((PASS + TP_PASS)) + FAIL=$((FAIL + TP_FAIL)) + if [ "$TP_FAIL" -gt 0 ]; then + echo " FAIL: $TP_FAIL top-p check(s) failed" + echo "$TP_OUTPUT" | grep "FAIL:" | head -4 + else + echo " PASS: top-p narrows the candidate set; out-of-range falls back to top-k" + fi + echo "" else echo "[47/47] Model roundtrip SKIPPED (binary built without EIGENSCRIPT_EXT_MODEL)" echo "" diff --git a/tests/test_eval_loss.sh b/tests/test_eval_loss.sh new file mode 100755 index 0000000..57d96eb --- /dev/null +++ b/tests/test_eval_loss.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# eigen_eval_loss: forward-only held-out cross-entropy. +# +# The property that pins this down without needing a trained model: an +# UNTRAINED model's output distribution is near-uniform over the vocabulary, +# so its cross-entropy must sit at ln(vocab_size) whatever the target is. The +# tiny fixture has vocab=8, so every loss must land near ln(8) = 2.0794. +# +# That single anchor catches the realistic failure modes at once -- a missing +# max-subtraction (overflow -> inf), summing the softmax over the wrong axis, +# indexing the wrong target, or returning logits instead of a loss would all +# miss ln(8). It also catches sign errors, since a cross-entropy is never +# negative. +# +# Runs only when the binary is built with EIGENSCRIPT_EXT_MODEL=1. + +set -u +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +EIGS="$TESTS_DIR/../src/eigenscript" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +fail() { echo " FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); } + +MODEL=/tmp/eigs_el_tiny.json +HARNESS=/tmp/eigs_el_harness.eigs +cleanup() { rm -f "$MODEL" "$HARNESS" /tmp/eigs_el_*.log; } +trap cleanup EXIT + +if ! "$EIGS" "$TESTS_DIR/gen_tiny_model.eigs" > "$MODEL" 2>/tmp/eigs_el_gen.log; then + fail "EL00 generate tiny model" "see /tmp/eigs_el_gen.log" + echo "EVALLOSS: 0 passed, 1 failed" + exit 1 +fi + +cat > "$HARNESS" </dev/null) +LOSSES=$(printf '%s\n' "$OUT" | sed -n 's/^L //p') + +if [ -z "$LOSSES" ]; then + fail "EL01 eigen_eval_loss returned values" "no output" + echo "EVALLOSS: $PASS passed, $((FAIL+1)) failed" + exit 1 +fi +ok "EL01 eigen_eval_loss returned a loss for every target" + +# All 8 losses within 0.35 nats of ln(8). The band is wide enough for the +# fixture's small random weights to tilt the distribution, tight enough that +# any structural error lands far outside it. +BAD=$(printf '%s\n' "$LOSSES" | awk ' + BEGIN { ln8 = 2.0794415; bad = 0 } + { d = $1 - ln8; if (d < 0) d = -d; if (d > 0.35) bad++ } + END { print bad }') +if [ "$BAD" = "0" ]; then + ok "EL02 untrained losses sit at ln(vocab) (all 8 within 0.35 of 2.0794)" +else + fail "EL02 untrained losses deviate from ln(vocab)" "$BAD of 8 outside band" + echo " losses: $(printf '%s ' $LOSSES)" +fi + +NONNEG=$(printf '%s\n' "$LOSSES" | awk '{ if ($1 < 0) n++ } END { print n+0 }') +if [ "$NONNEG" = "0" ]; then + ok "EL03 no negative cross-entropy" +else + fail "EL03 negative cross-entropy returned" "$NONNEG value(s) < 0" +fi + +OOR=$(printf '%s\n' "$OUT" | sed -n 's/^OOR //p') +NEG=$(printf '%s\n' "$OUT" | sed -n 's/^NEG //p') +if [ "${OOR%%.*}" = "-1" ] && [ "${NEG%%.*}" = "-1" ]; then + ok "EL04 out-of-range targets rejected with the -1 sentinel" +else + fail "EL04 out-of-range target not rejected" "target=999 -> '$OOR', target=-1 -> '$NEG'" +fi + +echo "EVALLOSS: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/tests/test_top_p.sh b/tests/test_top_p.sh new file mode 100755 index 0000000..6888873 --- /dev/null +++ b/tests/test_top_p.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# eigen_generate top-p (nucleus) sampling. +# +# Testing this needs care: generation draws from a global rand(), so two calls +# with the SAME policy already disagree on most positions (measured 3/40 +# identical). Comparing two token sequences therefore proves nothing about the +# sampling policy -- an earlier version of this check "verified" top-p by +# observing 0/40 matches, which is indistinguishable from the same-policy +# control. +# +# What IS robust is the statistical signature. On a near-uniform (untrained) +# model, top-p keeps every token up to a cumulative mass of p while top-k keeps +# a fixed few, so the two policies emit visibly different numbers of DISTINCT +# tokens over a long generation. That is a property of the candidate set, not +# of any particular rand() draw. On the vocab=1109 model the split is 61 vs 257 +# distinct in 300 tokens. +# +# Runs only when the binary is built with EIGENSCRIPT_EXT_MODEL=1. + +set -u +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +EIGS="$TESTS_DIR/../src/eigenscript" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +fail() { echo " FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); } + +MODEL=/tmp/eigs_tp_tiny.json +HARNESS=/tmp/eigs_tp_harness.eigs +cleanup() { rm -f "$MODEL" "$HARNESS" /tmp/eigs_tp_*.log; } +trap cleanup EXIT + +if ! "$EIGS" "$TESTS_DIR/gen_tiny_model.eigs" > "$MODEL" 2>/tmp/eigs_tp_gen.log; then + fail "TP00 generate tiny model" "see /tmp/eigs_tp_gen.log" + echo "TOPP: 0 passed, 1 failed" + exit 1 +fi + +# vocab=8, so the top-k=40 default clamps to "keep everything" and a restrictive +# top-p must keep strictly fewer. 400 tokens makes the distinct counts stable. +cat > "$HARNESS" <<'EIGS' +ck is args of null +r is eigen_model_load of ck[0] +define ndistinct(ids) as: + seen is zeros of 64 + d is 0 + for i in range of (len of ids): + t is ids[i] + if t >= 0: + if t < 64: + if seen[t] == 0: + seen[t] is 1 + d is d + 1 + return d +a is eigen_generate of [[1, 2, 3], 1.0, 400] +b is eigen_generate of [[1, 2, 3], 1.0, 400, 0.3] +print of ("K " + (str of (ndistinct of a))) +print of ("P " + (str of (ndistinct of b))) +print of ("PLEN " + (str of (len of b))) +EIGS + +OUT=$("$EIGS" "$HARNESS" "$MODEL" 2>/dev/null) +K=$(printf '%s\n' "$OUT" | sed -n 's/^K //p') +P=$(printf '%s\n' "$OUT" | sed -n 's/^P //p') +PLEN=$(printf '%s\n' "$OUT" | sed -n 's/^PLEN //p') + +if [ -z "$K" ] || [ -z "$P" ]; then + fail "TP01 both sampling policies produced output" "K='$K' P='$P'" + echo "TOPP: $PASS passed, $((FAIL)) failed" + exit 1 +fi +ok "TP01 both sampling policies produced output" + +if [ "$PLEN" = "400" ]; then + ok "TP02 top-p generated the requested token count" +else + fail "TP02 top-p token count" "expected 400, got $PLEN" +fi + +# The load-bearing assertion: a restrictive nucleus must narrow the candidate +# set. If top_p were ignored, this collapses to the same policy twice and the +# counts land equal. +if [ "$P" -lt "$K" ]; then + ok "TP03 top-p=0.3 narrows the candidate set vs top-k ($P < $K distinct)" +else + fail "TP03 top-p did not narrow the candidate set" "top-k=$K distinct, top-p=$P distinct" +fi + +# top_p outside (0,1) must fall back to the historical top-k path rather than +# emptying the candidate set or dividing by zero. +cat > "$HARNESS" <<'EIGS' +ck is args of null +r is eigen_model_load of ck[0] +for p in [0.0, 1.0, 2.0, 0 - 1.0]: + g is eigen_generate of [[1, 2, 3], 1.0, 20, p] + print of ("N " + (str of (len of g))) +EIGS +OOR=$("$EIGS" "$HARNESS" "$MODEL" 2>/dev/null | sed -n 's/^N //p' | sort -u) +if [ "$OOR" = "20" ]; then + ok "TP04 out-of-range top_p falls back to top-k and still generates" +else + fail "TP04 out-of-range top_p broke generation" "lengths: $(printf '%s ' $OOR)" +fi + +echo "TOPP: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]