Expose max_new_tokens run option

bindings/python/src/transcribe_cpp/__init__.py

@@ -582,12 +582,17 @@ def _stream_update_from(u) -> StreamUpdate:
 
 
 def _build_run_params(task, language, target_language, timestamps,
-                      keep_special_tags, spec_k_drafts):
+                      keep_special_tags, spec_k_drafts, max_new_tokens):
     if not isinstance(spec_k_drafts, int) or spec_k_drafts < -1:
         raise InvalidArgument(
             f"spec_k_drafts must be -1 (family default), 0 (disabled), or a "
             f"positive draft length; got {spec_k_drafts!r}"
         )
+    if not isinstance(max_new_tokens, int) or max_new_tokens < -1:
+        raise InvalidArgument(
+            f"max_new_tokens must be -1 (family default), 0 (family default), "
+            f"or a positive generation budget; got {max_new_tokens!r}"
+        )
     params = _RunParams()
     _lib.transcribe_run_params_init(_byref(params))
     params.task = _enum(_TASKS, task, "task")
@@ -596,6 +601,7 @@ def _build_run_params(task, language, target_language, timestamps,
     params.target_language = target_language.encode("utf-8") if target_language else None
     params.keep_special_tags = keep_special_tags
     params.spec_k_drafts = spec_k_drafts
+    params.max_new_tokens = max_new_tokens
     return params
 
 
@@ -984,6 +990,7 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe",
             timestamps: Timestamps = "auto",
             keep_special_tags: bool = False,
             spec_k_drafts: int = -1,
+            max_new_tokens: int = -1,
             family: FamilyExtension | None = None) -> Result:
         """Transcribe 16 kHz mono float32 PCM and return a materialized Result.
 
@@ -992,14 +999,17 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe",
         ``spec_k_drafts`` tunes speculative decoding on models whose
         capabilities advertise ``supports_spec_decode`` (-1 = family default,
         0 = disabled, >0 = draft length; silently ignored elsewhere).
+        ``max_new_tokens`` overrides the family default autoregressive output
+        budget where supported (-1 = family default).
 
         On ``Aborted`` (via :meth:`cancel`) and ``OutputTruncated`` the
         partial transcript is preserved and attached to the exception as
         ``partial_result``."""
         self._cancel.clear()
         array, n_samples = _pcm_to_carray(pcm)
         params = _build_run_params(task, language, target_language, timestamps,
-                                   keep_special_tags, spec_k_drafts)
+                                   keep_special_tags, spec_k_drafts,
+                                   max_new_tokens)
         ext = self._resolve_family(family, "run") if family is not None else None
         if ext is not None:
             params.family = ctypes.cast(
@@ -1020,6 +1030,7 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe",
                   timestamps: Timestamps = "auto",
                   keep_special_tags: bool = False,
                   spec_k_drafts: int = -1,
+                  max_new_tokens: int = -1,
                   family: FamilyExtension | None = None,
                   return_exceptions: bool = False) -> list[Result | TranscribeError]:
         """Transcribe several utterances in one dispatch — one Result each.
@@ -1054,7 +1065,8 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe",
             counts[k] = n
 
         params = _build_run_params(task, language, target_language, timestamps,
-                                   keep_special_tags, spec_k_drafts)
+                                   keep_special_tags, spec_k_drafts,
+                                   max_new_tokens)
         ext = self._resolve_family(family, "run") if family is not None else None
         if ext is not None:
             params.family = ctypes.cast(
@@ -1119,7 +1131,7 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None,
         # spec_k_drafts is an offline-decode knob; streaming always uses the
         # family default (-1).
         run_params = _build_run_params(task, language, target_language, timestamps,
-                                       keep_special_tags, -1)
+                                       keep_special_tags, -1, -1)
         sp = _StreamParams()
         _lib.transcribe_stream_params_init(_byref(sp))
         sp.commit_policy = _enum(_COMMIT_POLICIES, commit_policy, "commit_policy")

bindings/python/src/transcribe_cpp/_generated.py

@@ -13,7 +13,7 @@
 # Stable digest of the ABI surface below (structs, enums, macros, layout,
 # prototypes). A native provider package echoes this back so the API
 # package can reject an ABI-mismatched provider before dlopen.
-PUBLIC_HEADER_HASH = "86b16dd97ad1cb58"
+PUBLIC_HEADER_HASH = "a9feb7c785e278dc"
 
 # === enum constants ===
 TRANSCRIBE_OK = 0
@@ -152,7 +152,7 @@ class transcribe_whisper_chunk_trace(_c.Structure):
 transcribe_backend_device._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)]
 transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("gpu_device", _c.c_int)]
 transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)]
-transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32)]
+transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32), ("max_new_tokens", _c.c_int32)]
 transcribe_capabilities._fields_ = [("struct_size", _c.c_uint64), ("native_sample_rate", _c.c_int32), ("n_languages", _c.c_int), ("languages", _c.POINTER(_c.c_char_p)), ("max_timestamp_kind", _c.c_int), ("supports_language_detect", _c.c_bool), ("supports_translate", _c.c_bool), ("supports_streaming", _c.c_bool), ("supports_spec_decode", _c.c_bool), ("max_audio_ms", _c.c_int64), ("n_translate_target_languages", _c.c_int), ("translate_target_languages", _c.POINTER(_c.c_char_p))]
 transcribe_session_limits._fields_ = [("struct_size", _c.c_uint64), ("effective_n_ctx", _c.c_int32), ("effective_max_audio_ms", _c.c_int64), ("max_kv_bytes", _c.c_int64)]
 transcribe_stream_params._fields_ = [("struct_size", _c.c_uint64), ("family", _c.POINTER(transcribe_ext)), ("commit_policy", _c.c_int), ("stable_prefix_agreement_n", _c.c_uint32)]
@@ -194,7 +194,7 @@ class transcribe_whisper_chunk_trace(_c.Structure):
     'transcribe_backend_device': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}},
     'transcribe_model_load_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'gpu_device': 12}},
     'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}},
-    'transcribe_run_params': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56}},
+    'transcribe_run_params': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56, 'max_new_tokens': 60}},
     'transcribe_capabilities': {'size': 56, 'align': 8, 'offsets': {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48}},
     'transcribe_session_limits': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 'effective_n_ctx': 8, 'effective_max_audio_ms': 16, 'max_kv_bytes': 24}},
     'transcribe_stream_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'family': 8, 'commit_policy': 16, 'stable_prefix_agreement_n': 20}},

bindings/rust/sys/src/transcribe_sys.rs

@@ -1,11 +1,11 @@
 // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h
 // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`.
-// Pinned to include/transcribe.abihash = 86b16dd97ad1cb58
+// Pinned to include/transcribe.abihash = a9feb7c785e278dc
 
 /// The public-ABI digest these bindings were generated against
 /// (sha256/16 over the normalized FFI surface). The load-time version
 /// gate and the CI drift check both anchor on this value.
-pub const PUBLIC_HEADER_HASH: &str = "86b16dd97ad1cb58";
+pub const PUBLIC_HEADER_HASH: &str = "a9feb7c785e278dc";
 
 /* automatically generated by rust-bindgen 0.72.1 */
 
@@ -329,6 +329,7 @@ pub struct transcribe_run_params {
     pub keep_special_tags: bool,
     pub family: *const transcribe_ext,
     pub spec_k_drafts: i32,
+    pub max_new_tokens: i32,
 }
 #[allow(clippy::unnecessary_operation, clippy::identity_op)]
 const _: () = {
@@ -355,6 +356,8 @@ const _: () = {
         [::std::mem::offset_of!(transcribe_run_params, family) - 48usize];
     ["Offset of field: transcribe_run_params::spec_k_drafts"]
         [::std::mem::offset_of!(transcribe_run_params, spec_k_drafts) - 56usize];
+    ["Offset of field: transcribe_run_params::max_new_tokens"]
+        [::std::mem::offset_of!(transcribe_run_params, max_new_tokens) - 60usize];
 };
 unsafe extern "C" {
     pub fn transcribe_run_params_init(params: *mut transcribe_run_params);

bindings/rust/transcribe-cpp/src/session.rs

@@ -38,6 +38,8 @@ pub struct RunOptions {
     pub keep_special_tags: bool,
     /// Speculative-decode draft length. `-1` = family default, `0` = disabled.
     pub spec_k_drafts: i32,
+    /// Maximum transcript tokens to generate. `-1` = family default.
+    pub max_new_tokens: i32,
     /// Optional family-specific run extension (e.g. whisper decode knobs).
     pub family: Option<RunExtension>,
 }
@@ -53,6 +55,7 @@ impl Default for RunOptions {
             target_language: None,
             keep_special_tags: false,
             spec_k_drafts: -1,
+            max_new_tokens: -1,
             family: None,
         }
     }
@@ -412,6 +415,7 @@ fn build_run_params(o: &RunOptions) -> Result<RunParamsBundle> {
     params.itn = o.itn.to_raw();
     params.keep_special_tags = o.keep_special_tags;
     params.spec_k_drafts = o.spec_k_drafts;
+    params.max_new_tokens = o.max_new_tokens;
 
     let lang = o.language.as_deref().map(CString::new).transpose()?;
     let target = o.target_language.as_deref().map(CString::new).transpose()?;

bindings/swift/Sources/TranscribeCpp/ABIHash.swift

@@ -13,7 +13,7 @@ import CTranscribe
 extension Transcribe {
     /// sha256/16 of the normalized public FFI surface, pinned to the value in
     /// include/transcribe.abihash at the time this binding was last reviewed.
-    public static let pinnedHeaderHash = "86b16dd97ad1cb58"
+    public static let pinnedHeaderHash = "a9feb7c785e278dc"
 
     /// The public-ABI digest this binding was reviewed against (16 hex chars).
     public static func headerHash() -> String { pinnedHeaderHash }

bindings/swift/Sources/TranscribeCpp/Options.swift

@@ -121,6 +121,8 @@ public struct RunOptions: Sendable {
     public var keepSpecialTags: Bool
     /// Speculative-decode draft length: -1 = family default, 0 = disabled.
     public var specKDrafts: Int32
+    /// Maximum transcript tokens to generate: -1 = family default.
+    public var maxNewTokens: Int32
     /// Family-specific run extension (whisper run options); M3.
     public var family: RunExtension?
 
@@ -136,6 +138,7 @@ public struct RunOptions: Sendable {
         targetLanguage: String? = nil,
         keepSpecialTags: Bool = false,
         specKDrafts: Int32 = -1,
+        maxNewTokens: Int32 = -1,
         family: RunExtension? = nil
     ) {
         self.task = task
@@ -146,6 +149,7 @@ public struct RunOptions: Sendable {
         self.targetLanguage = targetLanguage
         self.keepSpecialTags = keepSpecialTags
         self.specKDrafts = specKDrafts
+        self.maxNewTokens = maxNewTokens
         self.family = family
     }
 
@@ -161,6 +165,7 @@ public struct RunOptions: Sendable {
         params.itn = itn.cValue
         params.keep_special_tags = keepSpecialTags
         params.spec_k_drafts = specKDrafts
+        params.max_new_tokens = maxNewTokens
         return try withOptionalCString(language) { lang in
             params.language = lang
             return try withOptionalCString(targetLanguage) { tgt in

bindings/typescript/src/_generated.ts

@@ -11,7 +11,7 @@
 // Stable digest of the ABI surface (structs, enums, macros, layout,
 // prototypes), computed by the Python oracle and pinned here so a header
 // ABI change turns this binding's drift check red for conscious review.
-export const PUBLIC_HEADER_HASH = "86b16dd97ad1cb58";
+export const PUBLIC_HEADER_HASH = "a9feb7c785e278dc";
 
 // === enum constants ===
 export const TRANSCRIBE_OK = 0;
@@ -110,7 +110,7 @@ export const STRUCT_LAYOUT: Record<string, StructLayout> = {
   'transcribe_backend_device': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} },
   'transcribe_model_load_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'gpu_device': 12} },
   'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} },
-  'transcribe_run_params': { size: 64, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56} },
+  'transcribe_run_params': { size: 64, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56, 'max_new_tokens': 60} },
   'transcribe_capabilities': { size: 56, align: 8, offsets: {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48} },
   'transcribe_session_limits': { size: 32, align: 8, offsets: {'struct_size': 0, 'effective_n_ctx': 8, 'effective_max_audio_ms': 16, 'max_kv_bytes': 24} },
   'transcribe_stream_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'family': 8, 'commit_policy': 16, 'stable_prefix_agreement_n': 20} },
@@ -152,7 +152,7 @@ export function defineTypes(koffi: any): Record<string, any> {
   T['transcribe_backend_device'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' });
   T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', gpu_device: 'int' });
   T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' });
-  T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t' });
+  T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t', max_new_tokens: 'int32_t' });
   T['transcribe_capabilities'] = koffi.struct({ struct_size: 'uint64_t', native_sample_rate: 'int32_t', n_languages: 'int', languages: 'void *', max_timestamp_kind: 'int', supports_language_detect: 'bool', supports_translate: 'bool', supports_streaming: 'bool', supports_spec_decode: 'bool', max_audio_ms: 'int64_t', n_translate_target_languages: 'int', translate_target_languages: 'void *' });
   T['transcribe_session_limits'] = koffi.struct({ struct_size: 'uint64_t', effective_n_ctx: 'int32_t', effective_max_audio_ms: 'int64_t', max_kv_bytes: 'int64_t' });
   T['transcribe_stream_params'] = koffi.struct({ struct_size: 'uint64_t', family: 'void *', commit_policy: 'int', stable_prefix_agreement_n: 'uint32_t' });

bindings/typescript/src/index.ts

@@ -776,6 +776,7 @@ export class Session {
     if (opts.keepSpecialTags !== undefined)
       p.keep_special_tags = opts.keepSpecialTags;
     if (opts.specKDrafts !== undefined) p.spec_k_drafts = opts.specKDrafts;
+    if (opts.maxNewTokens !== undefined) p.max_new_tokens = opts.maxNewTokens;
     if (opts.family)
       p.family = buildFamily(n, this.#model.handle, opts.family, "run");
     return p;

bindings/typescript/src/types.ts

@@ -136,6 +136,8 @@ export interface TranscribeOptions {
   keepSpecialTags?: boolean;
   /** Speculative-decode draft count; -1 = family default. */
   specKDrafts?: number;
+  /** Maximum transcript tokens to generate; -1 = family default. */
+  maxNewTokens?: number;
   /** Cancel the run cooperatively. */
   signal?: AbortSignal;
   /** A run-slot family extension (e.g. whisper). */

include/transcribe.abihash

@@ -1 +1 @@
-86b16dd97ad1cb58
+a9feb7c785e278dc

include/transcribe.h

@@ -1044,6 +1044,19 @@ struct transcribe_run_params {
      *   to know whether the field will take effect.
      */
     int32_t spec_k_drafts;
+
+    /*
+     * max_new_tokens: maximum autoregressive transcript tokens to generate
+     * before reporting TRANSCRIBE_ERR_OUTPUT_TRUNCATED.
+     *
+     *   Convention:
+     *     -1: family default.
+     *      0: family default.
+     *     >0: requested per-run generation budget, clamped by the model's
+     *         actual decoder context. Families without an autoregressive
+     *         output budget ignore this field.
+     */
+    int32_t max_new_tokens;
 };
 
 TRANSCRIBE_API void transcribe_run_params_init(struct transcribe_run_params * params);

src/arch/canary/model.cpp

@@ -25,6 +25,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
@@ -226,6 +227,17 @@ constexpr float kBnEps = 1e-5f;
 // FastConformer pre-encode downsamples time via stride-2, kernel-3, pad-1
 // convs; each stage maps T_in -> floor((T_in-1)/2)+1. We fold that exact
 // per-stage recurrence so the prediction matches the graph's T_enc.
+constexpr int k_default_max_new = 512;
+
+int canary_max_new_tokens(const transcribe_run_params * params) {
+    if (params != nullptr &&
+        params->struct_size >= offsetof(transcribe_run_params, max_new_tokens) + sizeof(params->max_new_tokens) &&
+        params->max_new_tokens > 0) {
+        return params->max_new_tokens;
+    }
+    return k_default_max_new;
+}
+
 int canary_predict_t_enc(int mel_n_frames, int subsampling_factor) {
     if (mel_n_frames <= 0 || subsampling_factor <= 0) {
         return 0;
@@ -1094,7 +1106,7 @@ transcribe_status run(transcribe_session *          session,
         cc->clear_result();
 
         const int eos_id     = cm->hparams.eos_token_id;
-        const int max_tokens = std::min(512, cc->kv_cache.n_ctx - prompt_len);
+        const int max_tokens = std::min(canary_max_new_tokens(params), cc->kv_cache.n_ctx - prompt_len);
 
         int next_token = 0;
         if (prompt_skip_softmax && db.argmax_out != nullptr) {
@@ -1610,7 +1622,7 @@ transcribe_status run_batch(transcribe_session *          session,
     }
 
     // Batched KV cache.
-    const int max_new  = 512;
+    const int max_new  = canary_max_new_tokens(params);
     int       max_n_kv = 1024;
     while (max_n_kv < prompt_len + max_new) {
         max_n_kv *= 2;