diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 2398aaf4..b87add9d 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -582,12 +582,17 @@ def _stream_update_from(u) -> StreamUpdate: def _build_run_params(task, language, target_language, timestamps, - keep_special_tags, spec_k_drafts): + keep_special_tags, spec_k_drafts, max_new_tokens): if not isinstance(spec_k_drafts, int) or spec_k_drafts < -1: raise InvalidArgument( f"spec_k_drafts must be -1 (family default), 0 (disabled), or a " f"positive draft length; got {spec_k_drafts!r}" ) + if not isinstance(max_new_tokens, int) or max_new_tokens < -1: + raise InvalidArgument( + f"max_new_tokens must be -1 (family default), 0 (family default), " + f"or a positive generation budget; got {max_new_tokens!r}" + ) params = _RunParams() _lib.transcribe_run_params_init(_byref(params)) params.task = _enum(_TASKS, task, "task") @@ -596,6 +601,7 @@ def _build_run_params(task, language, target_language, timestamps, params.target_language = target_language.encode("utf-8") if target_language else None params.keep_special_tags = keep_special_tags params.spec_k_drafts = spec_k_drafts + params.max_new_tokens = max_new_tokens return params @@ -984,6 +990,7 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, + max_new_tokens: int = -1, family: FamilyExtension | None = None) -> Result: """Transcribe 16 kHz mono float32 PCM and return a materialized Result. @@ -992,6 +999,8 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", ``spec_k_drafts`` tunes speculative decoding on models whose capabilities advertise ``supports_spec_decode`` (-1 = family default, 0 = disabled, >0 = draft length; silently ignored elsewhere). + ``max_new_tokens`` overrides the family default autoregressive output + budget where supported (-1 = family default). On ``Aborted`` (via :meth:`cancel`) and ``OutputTruncated`` the partial transcript is preserved and attached to the exception as @@ -999,7 +1008,8 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", self._cancel.clear() array, n_samples = _pcm_to_carray(pcm) params = _build_run_params(task, language, target_language, timestamps, - keep_special_tags, spec_k_drafts) + keep_special_tags, spec_k_drafts, + max_new_tokens) ext = self._resolve_family(family, "run") if family is not None else None if ext is not None: params.family = ctypes.cast( @@ -1020,6 +1030,7 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, + max_new_tokens: int = -1, family: FamilyExtension | None = None, return_exceptions: bool = False) -> list[Result | TranscribeError]: """Transcribe several utterances in one dispatch — one Result each. @@ -1054,7 +1065,8 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", counts[k] = n params = _build_run_params(task, language, target_language, timestamps, - keep_special_tags, spec_k_drafts) + keep_special_tags, spec_k_drafts, + max_new_tokens) ext = self._resolve_family(family, "run") if family is not None else None if ext is not None: params.family = ctypes.cast( @@ -1119,7 +1131,7 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, # spec_k_drafts is an offline-decode knob; streaming always uses the # family default (-1). run_params = _build_run_params(task, language, target_language, timestamps, - keep_special_tags, -1) + keep_special_tags, -1, -1) sp = _StreamParams() _lib.transcribe_stream_params_init(_byref(sp)) sp.commit_policy = _enum(_COMMIT_POLICIES, commit_policy, "commit_policy") diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 3862bd79..9d06968c 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "86b16dd97ad1cb58" +PUBLIC_HEADER_HASH = "a9feb7c785e278dc" # === enum constants === TRANSCRIBE_OK = 0 @@ -152,7 +152,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_backend_device._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("gpu_device", _c.c_int)] transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)] -transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32)] +transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32), ("max_new_tokens", _c.c_int32)] transcribe_capabilities._fields_ = [("struct_size", _c.c_uint64), ("native_sample_rate", _c.c_int32), ("n_languages", _c.c_int), ("languages", _c.POINTER(_c.c_char_p)), ("max_timestamp_kind", _c.c_int), ("supports_language_detect", _c.c_bool), ("supports_translate", _c.c_bool), ("supports_streaming", _c.c_bool), ("supports_spec_decode", _c.c_bool), ("max_audio_ms", _c.c_int64), ("n_translate_target_languages", _c.c_int), ("translate_target_languages", _c.POINTER(_c.c_char_p))] transcribe_session_limits._fields_ = [("struct_size", _c.c_uint64), ("effective_n_ctx", _c.c_int32), ("effective_max_audio_ms", _c.c_int64), ("max_kv_bytes", _c.c_int64)] transcribe_stream_params._fields_ = [("struct_size", _c.c_uint64), ("family", _c.POINTER(transcribe_ext)), ("commit_policy", _c.c_int), ("stable_prefix_agreement_n", _c.c_uint32)] @@ -194,7 +194,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_backend_device': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, 'transcribe_model_load_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'gpu_device': 12}}, 'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}}, - 'transcribe_run_params': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56}}, + 'transcribe_run_params': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56, 'max_new_tokens': 60}}, 'transcribe_capabilities': {'size': 56, 'align': 8, 'offsets': {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48}}, 'transcribe_session_limits': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 'effective_n_ctx': 8, 'effective_max_audio_ms': 16, 'max_kv_bytes': 24}}, 'transcribe_stream_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'family': 8, 'commit_policy': 16, 'stable_prefix_agreement_n': 20}}, diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index d5a353db..57329902 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 86b16dd97ad1cb58 +// Pinned to include/transcribe.abihash = a9feb7c785e278dc /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "86b16dd97ad1cb58"; +pub const PUBLIC_HEADER_HASH: &str = "a9feb7c785e278dc"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -329,6 +329,7 @@ pub struct transcribe_run_params { pub keep_special_tags: bool, pub family: *const transcribe_ext, pub spec_k_drafts: i32, + pub max_new_tokens: i32, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { @@ -355,6 +356,8 @@ const _: () = { [::std::mem::offset_of!(transcribe_run_params, family) - 48usize]; ["Offset of field: transcribe_run_params::spec_k_drafts"] [::std::mem::offset_of!(transcribe_run_params, spec_k_drafts) - 56usize]; + ["Offset of field: transcribe_run_params::max_new_tokens"] + [::std::mem::offset_of!(transcribe_run_params, max_new_tokens) - 60usize]; }; unsafe extern "C" { pub fn transcribe_run_params_init(params: *mut transcribe_run_params); diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index e58a9265..84616250 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -38,6 +38,8 @@ pub struct RunOptions { pub keep_special_tags: bool, /// Speculative-decode draft length. `-1` = family default, `0` = disabled. pub spec_k_drafts: i32, + /// Maximum transcript tokens to generate. `-1` = family default. + pub max_new_tokens: i32, /// Optional family-specific run extension (e.g. whisper decode knobs). pub family: Option, } @@ -53,6 +55,7 @@ impl Default for RunOptions { target_language: None, keep_special_tags: false, spec_k_drafts: -1, + max_new_tokens: -1, family: None, } } @@ -412,6 +415,7 @@ fn build_run_params(o: &RunOptions) -> Result { params.itn = o.itn.to_raw(); params.keep_special_tags = o.keep_special_tags; params.spec_k_drafts = o.spec_k_drafts; + params.max_new_tokens = o.max_new_tokens; let lang = o.language.as_deref().map(CString::new).transpose()?; let target = o.target_language.as_deref().map(CString::new).transpose()?; diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index 0ad79fb9..f3bcc372 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "86b16dd97ad1cb58" + public static let pinnedHeaderHash = "a9feb7c785e278dc" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index 4401fda3..7de4d5cc 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -121,6 +121,8 @@ public struct RunOptions: Sendable { public var keepSpecialTags: Bool /// Speculative-decode draft length: -1 = family default, 0 = disabled. public var specKDrafts: Int32 + /// Maximum transcript tokens to generate: -1 = family default. + public var maxNewTokens: Int32 /// Family-specific run extension (whisper run options); M3. public var family: RunExtension? @@ -136,6 +138,7 @@ public struct RunOptions: Sendable { targetLanguage: String? = nil, keepSpecialTags: Bool = false, specKDrafts: Int32 = -1, + maxNewTokens: Int32 = -1, family: RunExtension? = nil ) { self.task = task @@ -146,6 +149,7 @@ public struct RunOptions: Sendable { self.targetLanguage = targetLanguage self.keepSpecialTags = keepSpecialTags self.specKDrafts = specKDrafts + self.maxNewTokens = maxNewTokens self.family = family } @@ -161,6 +165,7 @@ public struct RunOptions: Sendable { params.itn = itn.cValue params.keep_special_tags = keepSpecialTags params.spec_k_drafts = specKDrafts + params.max_new_tokens = maxNewTokens return try withOptionalCString(language) { lang in params.language = lang return try withOptionalCString(targetLanguage) { tgt in diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index bf1da0d9..d09dfbe2 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "86b16dd97ad1cb58"; +export const PUBLIC_HEADER_HASH = "a9feb7c785e278dc"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -110,7 +110,7 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_backend_device': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, 'transcribe_model_load_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'gpu_device': 12} }, 'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} }, - 'transcribe_run_params': { size: 64, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56} }, + 'transcribe_run_params': { size: 64, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56, 'max_new_tokens': 60} }, 'transcribe_capabilities': { size: 56, align: 8, offsets: {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48} }, 'transcribe_session_limits': { size: 32, align: 8, offsets: {'struct_size': 0, 'effective_n_ctx': 8, 'effective_max_audio_ms': 16, 'max_kv_bytes': 24} }, 'transcribe_stream_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'family': 8, 'commit_policy': 16, 'stable_prefix_agreement_n': 20} }, @@ -152,7 +152,7 @@ export function defineTypes(koffi: any): Record { T['transcribe_backend_device'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', gpu_device: 'int' }); T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' }); - T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t' }); + T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t', max_new_tokens: 'int32_t' }); T['transcribe_capabilities'] = koffi.struct({ struct_size: 'uint64_t', native_sample_rate: 'int32_t', n_languages: 'int', languages: 'void *', max_timestamp_kind: 'int', supports_language_detect: 'bool', supports_translate: 'bool', supports_streaming: 'bool', supports_spec_decode: 'bool', max_audio_ms: 'int64_t', n_translate_target_languages: 'int', translate_target_languages: 'void *' }); T['transcribe_session_limits'] = koffi.struct({ struct_size: 'uint64_t', effective_n_ctx: 'int32_t', effective_max_audio_ms: 'int64_t', max_kv_bytes: 'int64_t' }); T['transcribe_stream_params'] = koffi.struct({ struct_size: 'uint64_t', family: 'void *', commit_policy: 'int', stable_prefix_agreement_n: 'uint32_t' }); diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index d87440f2..9459cbf7 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -776,6 +776,7 @@ export class Session { if (opts.keepSpecialTags !== undefined) p.keep_special_tags = opts.keepSpecialTags; if (opts.specKDrafts !== undefined) p.spec_k_drafts = opts.specKDrafts; + if (opts.maxNewTokens !== undefined) p.max_new_tokens = opts.maxNewTokens; if (opts.family) p.family = buildFamily(n, this.#model.handle, opts.family, "run"); return p; diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 12d65131..1728ad2e 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -136,6 +136,8 @@ export interface TranscribeOptions { keepSpecialTags?: boolean; /** Speculative-decode draft count; -1 = family default. */ specKDrafts?: number; + /** Maximum transcript tokens to generate; -1 = family default. */ + maxNewTokens?: number; /** Cancel the run cooperatively. */ signal?: AbortSignal; /** A run-slot family extension (e.g. whisper). */ diff --git a/include/transcribe.abihash b/include/transcribe.abihash index dc4df617..a94a7d24 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -86b16dd97ad1cb58 +a9feb7c785e278dc diff --git a/include/transcribe.h b/include/transcribe.h index c79b02e9..a5a68804 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -1044,6 +1044,19 @@ struct transcribe_run_params { * to know whether the field will take effect. */ int32_t spec_k_drafts; + + /* + * max_new_tokens: maximum autoregressive transcript tokens to generate + * before reporting TRANSCRIBE_ERR_OUTPUT_TRUNCATED. + * + * Convention: + * -1: family default. + * 0: family default. + * >0: requested per-run generation budget, clamped by the model's + * actual decoder context. Families without an autoregressive + * output budget ignore this field. + */ + int32_t max_new_tokens; }; TRANSCRIBE_API void transcribe_run_params_init(struct transcribe_run_params * params); diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 84e02976..980f89da 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -226,6 +227,17 @@ constexpr float kBnEps = 1e-5f; // FastConformer pre-encode downsamples time via stride-2, kernel-3, pad-1 // convs; each stage maps T_in -> floor((T_in-1)/2)+1. We fold that exact // per-stage recurrence so the prediction matches the graph's T_enc. +constexpr int k_default_max_new = 512; + +int canary_max_new_tokens(const transcribe_run_params * params) { + if (params != nullptr && + params->struct_size >= offsetof(transcribe_run_params, max_new_tokens) + sizeof(params->max_new_tokens) && + params->max_new_tokens > 0) { + return params->max_new_tokens; + } + return k_default_max_new; +} + int canary_predict_t_enc(int mel_n_frames, int subsampling_factor) { if (mel_n_frames <= 0 || subsampling_factor <= 0) { return 0; @@ -1094,7 +1106,7 @@ transcribe_status run(transcribe_session * session, cc->clear_result(); const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = std::min(512, cc->kv_cache.n_ctx - prompt_len); + const int max_tokens = std::min(canary_max_new_tokens(params), cc->kv_cache.n_ctx - prompt_len); int next_token = 0; if (prompt_skip_softmax && db.argmax_out != nullptr) { @@ -1610,7 +1622,7 @@ transcribe_status run_batch(transcribe_session * session, } // Batched KV cache. - const int max_new = 512; + const int max_new = canary_max_new_tokens(params); int max_n_kv = 1024; while (max_n_kv < prompt_len + max_new) { max_n_kv *= 2; diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 6e382fbe..23da4203 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -82,9 +83,18 @@ constexpr const char k_default_variant[] = "qwen3-asr"; // transcript that fills the generation budget before end-of-stream is flagged // via transcribe_was_truncated(). -// Per-run generation budget (matches the reference dumper default). +// Default per-run generation budget (matches the reference dumper default). constexpr int k_max_new = 256; +int qwen3_max_new_tokens(const transcribe_run_params * params) { + if (params != nullptr && + params->struct_size >= offsetof(transcribe_run_params, max_new_tokens) + sizeof(params->max_new_tokens) && + params->max_new_tokens > 0) { + return params->max_new_tokens; + } + return k_max_new; +} + // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. int qwen3_context_ceiling(int32_t n_ctx_knob, const QwenAsrHParams & hp) { @@ -734,13 +744,14 @@ transcribe_status run(transcribe_session * session, // Input-length gate: audio + prompt + generation must fit the decoder // context window. Reject an over-length clip here, before prefill/decode. const int ceiling = qwen3_context_ceiling(cc->n_ctx, cm->hparams); - if (T_prompt + k_max_new > ceiling) { + const int max_new = qwen3_max_new_tokens(params); + if (T_prompt + max_new > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_enc, prefix_len + suffix_len, ceiling, T_prompt + max_new); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } @@ -749,7 +760,7 @@ transcribe_status run(transcribe_session * session, // graph's flash-attn path wants pow2 attention width). A pre-allocated // smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -879,7 +890,6 @@ transcribe_status run(transcribe_session * session, // Step loop. const int32_t eos_id = cm->hparams.eos_token_id; - const int32_t max_new = k_max_new; int cur_past = T_prompt; // Build the step graph ONCE and reuse every step, sized for the actual @@ -1550,7 +1560,7 @@ transcribe_status run_batch(transcribe_session * session, // Prompt length bound → max_n_kv and batched-cache n_ctx. Build and keep // each utterance's prompt token ids for the batched prefill. - const int max_new = 256; + const int max_new = qwen3_max_new_tokens(params); int max_T_prompt = 0; int prefix_len = 0; // Per-utterance terminal status for rejected rows. Defaults to INVALID_ARG; diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 6e254815..cc21ffb8 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -566,8 +566,9 @@ extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { return; } std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - p->spec_k_drafts = -1; // family default + p->struct_size = sizeof(*p); + p->spec_k_drafts = -1; // family default + p->max_new_tokens = -1; // family default // Default to AUTO ("richest the model supports", resolved per-family at // run time) rather than the memset NONE. p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO;