diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index b13a92ad..8967a178 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -582,7 +582,7 @@ def _stream_update_from(u) -> StreamUpdate: def _build_run_params(task, language, target_language, timestamps, - keep_special_tags, spec_k_drafts): + keep_special_tags, spec_k_drafts, context=None): if not isinstance(spec_k_drafts, int) or spec_k_drafts < -1: raise InvalidArgument( f"spec_k_drafts must be -1 (family default), 0 (disabled), or a " @@ -596,6 +596,7 @@ def _build_run_params(task, language, target_language, timestamps, params.target_language = target_language.encode("utf-8") if target_language else None params.keep_special_tags = keep_special_tags params.spec_k_drafts = spec_k_drafts + params.context = context.encode("utf-8") if context else None return params @@ -984,6 +985,7 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, + context: str | None = None, family: FamilyExtension | None = None) -> Result: """Transcribe 16 kHz mono float32 PCM and return a materialized Result. @@ -992,6 +994,10 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", ``spec_k_drafts`` tunes speculative decoding on models whose capabilities advertise ``supports_spec_decode`` (-1 = family default, 0 = disabled, >0 = draft length; silently ignored elsewhere). + ``context`` is free text (names, jargon, prior transcript) that + biases recognition on models advertising the ``initial_prompt`` + feature (whisper, qwen3_asr, funasr_nano); the output remains a + transcript, and unsupported models warn and ignore it. On ``Aborted`` (via :meth:`cancel`) and ``OutputTruncated`` the partial transcript is preserved and attached to the exception as @@ -999,7 +1005,7 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", self._cancel.clear() array, n_samples = _pcm_to_carray(pcm) params = _build_run_params(task, language, target_language, timestamps, - keep_special_tags, spec_k_drafts) + keep_special_tags, spec_k_drafts, context) ext = self._resolve_family(family, "run") if family is not None else None if ext is not None: params.family = ctypes.cast( @@ -1020,6 +1026,7 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, + context: str | None = None, family: FamilyExtension | None = None, return_exceptions: bool = False) -> list[Result | TranscribeError]: """Transcribe several utterances in one dispatch — one Result each. @@ -1054,7 +1061,7 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", counts[k] = n params = _build_run_params(task, language, target_language, timestamps, - keep_special_tags, spec_k_drafts) + keep_special_tags, spec_k_drafts, context) ext = self._resolve_family(family, "run") if family is not None else None if ext is not None: params.family = ctypes.cast( @@ -1107,6 +1114,7 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, target_language: str | None = None, timestamps: Timestamps = "none", keep_special_tags: bool = False, commit_policy: CommitPolicy = "auto", stable_prefix_agreement_n: int = 0, + context: str | None = None, family: FamilyExtension | None = None) -> Stream: """Begin streaming on this session and return a Stream to feed audio to. @@ -1119,7 +1127,7 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, # spec_k_drafts is an offline-decode knob; streaming always uses the # family default (-1). run_params = _build_run_params(task, language, target_language, timestamps, - keep_special_tags, -1) + keep_special_tags, -1, context) sp = _StreamParams() _lib.transcribe_stream_params_init(_byref(sp)) sp.commit_policy = _enum(_COMMIT_POLICIES, commit_policy, "commit_policy") @@ -1348,6 +1356,7 @@ def transcribe( timestamps: Timestamps = "auto", keep_special_tags: bool = False, spec_k_drafts: int = -1, + context: str | None = None, family: FamilyExtension | None = None, ) -> Result: """Transcribe *pcm* in one call and return a materialized Result. @@ -1362,7 +1371,7 @@ def transcribe( session_opts = dict(n_threads=n_threads, kv_type=kv_type, n_ctx=n_ctx) run_opts = dict(task=task, language=language, target_language=target_language, timestamps=timestamps, keep_special_tags=keep_special_tags, - spec_k_drafts=spec_k_drafts, family=family) + spec_k_drafts=spec_k_drafts, context=context, family=family) if isinstance(model, Model): with model.session(**session_opts) as session: diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 3862bd79..a4b0bb00 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "86b16dd97ad1cb58" +PUBLIC_HEADER_HASH = "3ba8dde028c487b6" # === enum constants === TRANSCRIBE_OK = 0 @@ -152,7 +152,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_backend_device._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("gpu_device", _c.c_int)] transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)] -transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32)] +transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32), ("context", _c.c_char_p)] transcribe_capabilities._fields_ = [("struct_size", _c.c_uint64), ("native_sample_rate", _c.c_int32), ("n_languages", _c.c_int), ("languages", _c.POINTER(_c.c_char_p)), ("max_timestamp_kind", _c.c_int), ("supports_language_detect", _c.c_bool), ("supports_translate", _c.c_bool), ("supports_streaming", _c.c_bool), ("supports_spec_decode", _c.c_bool), ("max_audio_ms", _c.c_int64), ("n_translate_target_languages", _c.c_int), ("translate_target_languages", _c.POINTER(_c.c_char_p))] transcribe_session_limits._fields_ = [("struct_size", _c.c_uint64), ("effective_n_ctx", _c.c_int32), ("effective_max_audio_ms", _c.c_int64), ("max_kv_bytes", _c.c_int64)] transcribe_stream_params._fields_ = [("struct_size", _c.c_uint64), ("family", _c.POINTER(transcribe_ext)), ("commit_policy", _c.c_int), ("stable_prefix_agreement_n", _c.c_uint32)] @@ -194,7 +194,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_backend_device': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, 'transcribe_model_load_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'gpu_device': 12}}, 'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}}, - 'transcribe_run_params': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56}}, + 'transcribe_run_params': {'size': 72, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56, 'context': 64}}, 'transcribe_capabilities': {'size': 56, 'align': 8, 'offsets': {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48}}, 'transcribe_session_limits': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 'effective_n_ctx': 8, 'effective_max_audio_ms': 16, 'max_kv_bytes': 24}}, 'transcribe_stream_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'family': 8, 'commit_policy': 16, 'stable_prefix_agreement_n': 20}}, diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 5da6ab48..6a4539b0 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -386,7 +386,7 @@ wheels = [ [[package]] name = "transcribe-cpp" -version = "0.0.1" +version = "0.1.1" source = { editable = "." } [package.optional-dependencies] @@ -402,8 +402,8 @@ test = [ requires-dist = [ { name = "numpy", marker = "extra == 'test'" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=7" }, - { name = "transcribe-cpp-native", specifier = "==0.0.1.*" }, - { name = "transcribe-cpp-native-cu12", marker = "extra == 'cu12'", specifier = "==0.0.1.*" }, + { name = "transcribe-cpp-native", specifier = "==0.1.1.*" }, + { name = "transcribe-cpp-native-cu12", marker = "extra == 'cu12'", specifier = "==0.1.1.*" }, ] [[package]] diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index d5a353db..8bc68612 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 86b16dd97ad1cb58 +// Pinned to include/transcribe.abihash = 3ba8dde028c487b6 /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "86b16dd97ad1cb58"; +pub const PUBLIC_HEADER_HASH: &str = "3ba8dde028c487b6"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -329,10 +329,11 @@ pub struct transcribe_run_params { pub keep_special_tags: bool, pub family: *const transcribe_ext, pub spec_k_drafts: i32, + pub context: *const ::std::os::raw::c_char, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of transcribe_run_params"][::std::mem::size_of::() - 64usize]; + ["Size of transcribe_run_params"][::std::mem::size_of::() - 72usize]; ["Alignment of transcribe_run_params"] [::std::mem::align_of::() - 8usize]; ["Offset of field: transcribe_run_params::struct_size"] @@ -355,6 +356,8 @@ const _: () = { [::std::mem::offset_of!(transcribe_run_params, family) - 48usize]; ["Offset of field: transcribe_run_params::spec_k_drafts"] [::std::mem::offset_of!(transcribe_run_params, spec_k_drafts) - 56usize]; + ["Offset of field: transcribe_run_params::context"] + [::std::mem::offset_of!(transcribe_run_params, context) - 64usize]; }; unsafe extern "C" { pub fn transcribe_run_params_init(params: *mut transcribe_run_params); diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index e58a9265..f54eb884 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -38,6 +38,11 @@ pub struct RunOptions { pub keep_special_tags: bool, /// Speculative-decode draft length. `-1` = family default, `0` = disabled. pub spec_k_drafts: i32, + /// Free-text recognition-biasing context (names, jargon, prior + /// transcript). Honored by models advertising the initial-prompt + /// feature (whisper, qwen3_asr, funasr_nano); the output remains a + /// transcript, and unsupported models warn and ignore it. + pub context: Option, /// Optional family-specific run extension (e.g. whisper decode knobs). pub family: Option, } @@ -53,6 +58,7 @@ impl Default for RunOptions { target_language: None, keep_special_tags: false, spec_k_drafts: -1, + context: None, family: None, } } @@ -144,7 +150,7 @@ impl Session { /// On an aborted or truncated decode the partial transcript is preserved /// on the returned [`Error::Aborted`] / [`Error::OutputTruncated`]. pub fn run(&mut self, pcm: &[f32], options: &RunOptions) -> Result { - let (params, _lang, _target, _family) = build_run_params(options)?; + let (params, _lang, _target, _context, _family) = build_run_params(options)?; let n = clamp_len(pcm.len())?; // The compute path is serialized per model; hold the lock for the native @@ -187,7 +193,7 @@ impl Session { pcms: &[&[f32]], options: &RunOptions, ) -> Result>> { - let (params, _lang, _target, _family) = build_run_params(options)?; + let (params, _lang, _target, _context, _family) = build_run_params(options)?; let ptrs: Vec<*const f32> = pcms.iter().map(|p| p.as_ptr()).collect(); let lens: Vec = pcms .iter() @@ -263,7 +269,7 @@ impl Session { /// Dropping the returned `Stream` abandons it and returns the session to /// idle. pub fn stream(&mut self, run: &RunOptions, stream: &StreamOptions) -> Result> { - let (run_params, _lang, _target, _family) = build_run_params(run)?; + let (run_params, _lang, _target, _context, _family) = build_run_params(run)?; let (stream_params, _stream_family) = build_stream_params(stream); { // Claim the model's compute lease for the whole stream lifetime: a @@ -396,6 +402,7 @@ type RunParamsBundle = ( sys::transcribe_run_params, Option, Option, + Option, Option, ); @@ -415,8 +422,10 @@ fn build_run_params(o: &RunOptions) -> Result { let lang = o.language.as_deref().map(CString::new).transpose()?; let target = o.target_language.as_deref().map(CString::new).transpose()?; + let context = o.context.as_deref().map(CString::new).transpose()?; params.language = lang.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()); params.target_language = target.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()); + params.context = context.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()); let family = o .family @@ -425,7 +434,7 @@ fn build_run_params(o: &RunOptions) -> Result { .transpose()?; params.family = family.as_ref().map_or(std::ptr::null(), |f| f.ext_ptr()); - Ok((params, lang, target, family)) + Ok((params, lang, target, context, family)) } /// PCM/utterance lengths cross the ABI as `int`; reject anything that overflows. diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index 0ad79fb9..9792f63d 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "86b16dd97ad1cb58" + public static let pinnedHeaderHash = "3ba8dde028c487b6" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index 4401fda3..ee064005 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -121,6 +121,11 @@ public struct RunOptions: Sendable { public var keepSpecialTags: Bool /// Speculative-decode draft length: -1 = family default, 0 = disabled. public var specKDrafts: Int32 + /// Free-text recognition-biasing context (names, jargon, prior + /// transcript). Honored by models advertising the initial-prompt + /// feature (whisper, qwen3_asr, funasr_nano); the output remains a + /// transcript, and unsupported models warn and ignore it. + public var context: String? /// Family-specific run extension (whisper run options); M3. public var family: RunExtension? @@ -136,6 +141,7 @@ public struct RunOptions: Sendable { targetLanguage: String? = nil, keepSpecialTags: Bool = false, specKDrafts: Int32 = -1, + context: String? = nil, family: RunExtension? = nil ) { self.task = task @@ -146,6 +152,7 @@ public struct RunOptions: Sendable { self.targetLanguage = targetLanguage self.keepSpecialTags = keepSpecialTags self.specKDrafts = specKDrafts + self.context = context self.family = family } @@ -165,9 +172,12 @@ public struct RunOptions: Sendable { params.language = lang return try withOptionalCString(targetLanguage) { tgt in params.target_language = tgt - return try withRunExtension(family) { ext in - params.family = ext - return try withUnsafePointer(to: ¶ms) { try body($0) } + return try withOptionalCString(context) { ctx in + params.context = ctx + return try withRunExtension(family) { ext in + params.family = ext + return try withUnsafePointer(to: ¶ms) { try body($0) } + } } } } diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index bf1da0d9..e2800941 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "86b16dd97ad1cb58"; +export const PUBLIC_HEADER_HASH = "3ba8dde028c487b6"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -110,7 +110,7 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_backend_device': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, 'transcribe_model_load_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'gpu_device': 12} }, 'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} }, - 'transcribe_run_params': { size: 64, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56} }, + 'transcribe_run_params': { size: 72, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'language': 24, 'target_language': 32, 'keep_special_tags': 40, 'family': 48, 'spec_k_drafts': 56, 'context': 64} }, 'transcribe_capabilities': { size: 56, align: 8, offsets: {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48} }, 'transcribe_session_limits': { size: 32, align: 8, offsets: {'struct_size': 0, 'effective_n_ctx': 8, 'effective_max_audio_ms': 16, 'max_kv_bytes': 24} }, 'transcribe_stream_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'family': 8, 'commit_policy': 16, 'stable_prefix_agreement_n': 20} }, @@ -152,7 +152,7 @@ export function defineTypes(koffi: any): Record { T['transcribe_backend_device'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', gpu_device: 'int' }); T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' }); - T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t' }); + T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t', context: 'char *' }); T['transcribe_capabilities'] = koffi.struct({ struct_size: 'uint64_t', native_sample_rate: 'int32_t', n_languages: 'int', languages: 'void *', max_timestamp_kind: 'int', supports_language_detect: 'bool', supports_translate: 'bool', supports_streaming: 'bool', supports_spec_decode: 'bool', max_audio_ms: 'int64_t', n_translate_target_languages: 'int', translate_target_languages: 'void *' }); T['transcribe_session_limits'] = koffi.struct({ struct_size: 'uint64_t', effective_n_ctx: 'int32_t', effective_max_audio_ms: 'int64_t', max_kv_bytes: 'int64_t' }); T['transcribe_stream_params'] = koffi.struct({ struct_size: 'uint64_t', family: 'void *', commit_policy: 'int', stable_prefix_agreement_n: 'uint32_t' }); diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index d87440f2..48582b6c 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -776,6 +776,7 @@ export class Session { if (opts.keepSpecialTags !== undefined) p.keep_special_tags = opts.keepSpecialTags; if (opts.specKDrafts !== undefined) p.spec_k_drafts = opts.specKDrafts; + if (opts.context !== undefined) p.context = opts.context; if (opts.family) p.family = buildFamily(n, this.#model.handle, opts.family, "run"); return p; @@ -872,6 +873,7 @@ export class Session { timestamps: opts.timestamps, keepSpecialTags: opts.keepSpecialTags, specKDrafts: -1, + context: opts.context, }); const sp: any = {}; F.streamParamsInit(sp); diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 12d65131..5ee7d03f 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -136,6 +136,11 @@ export interface TranscribeOptions { keepSpecialTags?: boolean; /** Speculative-decode draft count; -1 = family default. */ specKDrafts?: number; + /** Free-text recognition-biasing context (names, jargon, prior + * transcript). Honored by models advertising the initial-prompt + * feature (whisper, qwen3_asr, funasr_nano); the output remains a + * transcript, and unsupported models warn and ignore it. */ + context?: string; /** Cancel the run cooperatively. */ signal?: AbortSignal; /** A run-slot family extension (e.g. whisper). */ @@ -181,6 +186,8 @@ export interface StreamOptions { targetLanguage?: string; timestamps?: TimestampKind; keepSpecialTags?: boolean; + /** Free-text recognition-biasing context; see TranscribeOptions.context. */ + context?: string; commitPolicy?: CommitPolicy; stablePrefixAgreementN?: number; /** A stream-slot family extension (moonshine, parakeet, voxtral). */ diff --git a/docs/models/fun-asr-nano.md b/docs/models/fun-asr-nano.md index 07beb665..eeffc114 100644 --- a/docs/models/fun-asr-nano.md +++ b/docs/models/fun-asr-nano.md @@ -84,6 +84,12 @@ All Fun-ASR-Nano variants support: punctuation) via `--itn` on the CLI, or `transcribe_funasr_nano_params { use_itn = true }` via the library API. +- **Hotword / context biasing** via `--context` on the CLI or + `transcribe_run_params::context` in the API. The text fills the + hotword slot of the trained prompt template (upstream + `FunASRNano.get_prompt` hotwords path), so comma-separated + keyword lists match the trained shape best — e.g. + `--context "pull buoy, catch-up freestyle"`. What's not supported (consistent across the family): translation, real-time streaming, long-form chunking, timestamps, VAD, speaker diff --git a/docs/models/qwen3-asr.md b/docs/models/qwen3-asr.md index a5553c73..54adcc08 100644 --- a/docs/models/qwen3-asr.md +++ b/docs/models/qwen3-asr.md @@ -72,6 +72,12 @@ All Qwen3-ASR variants support: - **Transcription** of 16 kHz mono WAV input. - **Auto language detection** across 30 languages. +- **Context biasing** — Qwen3-ASR's trained "customized recognition" + channel. Pass names/jargon/prior context via `--context` (or + `transcribe_run_params::context`); the text rides the chat template's + system slot, exactly where the upstream reference puts its `context` + string. Keyword lists and free paragraphs both work; the output + remains a transcript of the audio. What's not supported (consistent across the family): translation, real-time streaming, VAD, speaker diarization, timestamps. See the diff --git a/docs/models/voxtral.md b/docs/models/voxtral.md index eb8e4435..38c4fd92 100644 --- a/docs/models/voxtral.md +++ b/docs/models/voxtral.md @@ -42,5 +42,10 @@ reports the exact per-session value. See the - The 24B is the larger sibling — same architecture, scaled decoder. It is a GPU-class model (BF16/F16 need ~50 GB) and should be run at **batch size ≤ 8**; see its [card](voxtral-small-24b-2507.md) for details. +- **No context biasing**: Voxtral's transcription template has no slot for + `transcribe_run_params::context` (a supplied context warns and is + ignored). The model's free-text instruct mode exists upstream but changes + the task rather than biasing recognition, so it is not mapped onto the + core context field; if exposed later it would be a voxtral run extension. - Upstream: [`mistralai/Voxtral-Mini-3B-2507`](https://huggingface.co/mistralai/Voxtral-Mini-3B-2507), [`mistralai/Voxtral-Small-24B-2507`](https://huggingface.co/mistralai/Voxtral-Small-24B-2507). diff --git a/docs/models/whisper.md b/docs/models/whisper.md index cb940ded..52e02470 100644 --- a/docs/models/whisper.md +++ b/docs/models/whisper.md @@ -91,6 +91,11 @@ section. All Whisper variants support: - **Transcription** of 16 kHz mono WAV input. +- **Context biasing** — pass names/jargon/prior-transcript text via + `--context` (or `transcribe_run_params::context`); it is injected as + Whisper previous-context after `<|startofprev|>`. The whisper run + extension's `initial_prompt` / `prompt_tokens` are the power-user + form of the same mechanism and win when both are set. - **Long-form audio** via 30-second chunked decoding with the prev-context window assembly described in the family doc. - **Segment timestamps** — the finest granularity the library emits for diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index c407f98b..8db3cd1f 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -131,6 +131,10 @@ struct cli_args { int gpu_device = 0; // --device N: 0 = auto, >0 = registry index transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; + // Generic recognition-biasing context (transcribe_run_params::context). + // Families without the INITIAL_PROMPT feature warn and ignore it. + std::string context; // --context TEXT + // Whisper-family knobs. Ignored for non-Whisper models. std::string initial_prompt; // --initial-prompt TEXT bool whisper_set = false; @@ -211,7 +215,11 @@ void print_usage(const char * argv0) { " --batch-jsonl output one JSON line per file (for batch)\n" " --batch-size N group N utterances into one transcribe_run_batch\n" " call (offline only; 0/1 = per-file serial loop)\n" - " --initial-prompt TEXT (whisper) initial prompt text for context biasing\n" + " --context TEXT recognition-biasing context (names, jargon,\n" + " prior transcript); models without the feature\n" + " warn and ignore it\n" + " --initial-prompt TEXT (whisper) initial prompt text for context biasing;\n" + " wins over --context on whisper\n" " --temperature F (whisper) tier-0 sampling temperature (default 0 = greedy)\n" " --condition-on-prev-tokens (whisper) carry prev-chunk tokens across chunks\n" " --prompt-condition T (whisper) prompt placement: first|all (default: first)\n" @@ -439,6 +447,12 @@ bool parse_args(int argc, char ** argv, cli_args & out) { std::fprintf(stderr, "error: --batch-size must be >= 0\n"); return false; } + } else if (a == "--context") { + const char * v = take_value(a.c_str()); + if (!v) { + return false; + } + out.context = v; } else if (a == "--initial-prompt") { const char * v = take_value(a.c_str()); if (!v) { @@ -698,6 +712,9 @@ int main(int argc, char ** argv) { if (args.canary_pnc_set) { rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; } + if (!args.context.empty()) { + rp.context = args.context.c_str(); + } // Whisper run extension. Allocated outside rp's scope so its // bytes outlive the per-file loop below; the library copies @@ -1075,6 +1092,9 @@ int main(int argc, char ** argv) { if (args.canary_pnc_set) { rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; } + if (!args.context.empty()) { + rp.context = args.context.c_str(); + } struct transcribe_whisper_run_ext wx; transcribe_whisper_run_ext_init(&wx); diff --git a/include/transcribe.abihash b/include/transcribe.abihash index dc4df617..bbb0b5aa 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -86b16dd97ad1cb58 +3ba8dde028c487b6 diff --git a/include/transcribe.h b/include/transcribe.h index 81b2d37b..ab39dbab 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -984,14 +984,14 @@ TRANSCRIBE_API void transcribe_session_params_init(struct transcribe_session_par * * target_language: target language for translation tasks, or NULL. * - * String-pointer lifetime (language / target_language): caller-owned, and - * the library copies what it needs before the API call returns. This holds - * for transcribe_run / transcribe_run_batch (synchronous) AND for - * transcribe_stream_begin: the dispatcher copies these strings into - * session-owned storage at begin, so the caller may free its params — - * including the strings — the moment begin returns, even though the stream - * keeps running. Bindings rely on this: ctypes/FFI-managed string buffers - * die when the begin wrapper returns. + * String-pointer lifetime (language / target_language / context): + * caller-owned, and the library copies what it needs before the API call + * returns. This holds for transcribe_run / transcribe_run_batch + * (synchronous) AND for transcribe_stream_begin: the dispatcher copies + * these strings into session-owned storage at begin, so the caller may + * free its params — including the strings — the moment begin returns, + * even though the stream keeps running. Bindings rely on this: + * ctypes/FFI-managed string buffers die when the begin wrapper returns. * * keep_special_tags: keep special vocabulary tags (e.g. <|...|>) in the * returned text fields. Default (false) strips them @@ -1044,6 +1044,43 @@ struct transcribe_run_params { * to know whether the field will take effect. */ int32_t spec_k_drafts; + + /* + * context: free-text that conditions recognition — names, jargon, + * hotwords, prior-transcript or domain context. NULL (default) or + * empty means no context. + * + * Contract: the output remains a faithful transcript of the audio. + * context biases WHAT the model hears (spelling of names, choice + * between homophones), never what it does — it is not an + * instruction channel, and families never route it anywhere that + * would change the task. Each family maps the string to its native + * mechanism: + * + * whisper previous-context prefix after <|startofprev|> + * (same path as transcribe_whisper_run_ext's + * initial_prompt; the ext, when present, wins — see + * include/transcribe/whisper.h for precedence). + * qwen3_asr system-message context slot (the model's + * trained biasing channel; accepts keyword lists + * or free paragraphs). + * funasr_nano hotword/context slot of the trained prompt + * template. Keyword-list-style text ("Foo, Bar, + * Baz") matches the trained shape best. + * + * A non-empty context against a model for which + * transcribe_model_supports(model, TRANSCRIBE_FEATURE_INITIAL_PROMPT) + * is false emits a WARN and proceeds without it (same advisory + * semantics as pnc/itn). + * + * Field-promotion rule (why this is core and e.g. voxtral's + * instruct mode is not): a knob enters transcribe_run_params when + * two or more families can honor the same caller intent WITHOUT + * changing the output contract. Anything that changes what the + * model does (free-form task instructions, chat) stays on the + * owning family's extension header. + */ + const char * context; }; TRANSCRIBE_API void transcribe_run_params_init(struct transcribe_run_params * params); @@ -1231,9 +1268,13 @@ TRANSCRIBE_API transcribe_status transcribe_model_get_capabilities(const struct * * Feature meanings: * - * INITIAL_PROMPT The model accepts a free-text or token - * prompt to bias decoding. Today: whisper - * only; reached via transcribe_whisper_run_ext. + * INITIAL_PROMPT The model honors transcribe_run_params::context + * — free text that biases recognition while the + * output remains a transcript. Today: whisper, + * qwen3_asr, funasr_nano. Whisper additionally + * accepts token-level prompts and decode knobs + * via transcribe_whisper_run_ext, which takes + * precedence over the core field. * * TEMPERATURE_FALLBACK The model runs a multi-tier temperature loop * with metric-driven fallback. Today: whisper. diff --git a/include/transcribe/whisper.h b/include/transcribe/whisper.h index f40d0d66..c5884c58 100644 --- a/include/transcribe/whisper.h +++ b/include/transcribe/whisper.h @@ -102,6 +102,11 @@ struct transcribe_whisper_run_ext { * rejected with TRANSCRIBE_ERR_INVALID_ARG, mirroring HF's * own check. * + * Else: the core transcribe_run_params::context string, when set, + * feeds this same path (identical tokenization and special-token + * rejection). The ext fields, being the more specific surface, + * always win over the core field. + * * Else: no initial prompt. */ const char * initial_prompt; diff --git a/src/arch/funasr_nano/capabilities.cpp b/src/arch/funasr_nano/capabilities.cpp index b3c163c7..f52ac995 100644 --- a/src/arch/funasr_nano/capabilities.cpp +++ b/src/arch/funasr_nano/capabilities.cpp @@ -24,6 +24,11 @@ void apply_family_invariants(transcribe_model & model) { // runtime toggle. transcribe::set_feature(&model, TRANSCRIBE_FEATURE_CANCELLATION, true); transcribe::set_feature(&model, TRANSCRIBE_FEATURE_ITN, true); + + // transcribe_run_params::context is honored: the string fills the + // hotword/context slot of the trained prompt template (the reference's + // FunASRNano.get_prompt hotwords path — 热词列表:[...]). + transcribe::set_feature(&model, TRANSCRIBE_FEATURE_INITIAL_PROMPT, true); } } // namespace transcribe::funasr_nano diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index c1aa74c1..0e4460cb 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -147,15 +147,49 @@ transcribe_status resolve_chat_tokens(const transcribe::Tokenizer & tok, ChatTok } // Build the language/itn prompt text that the reference's -// FunASRNano.get_prompt produces. hotwords-empty path only. -std::string build_funasr_prompt_text(const char * lang, bool use_itn) { +// FunASRNano.get_prompt produces. A non-empty `context` renders the +// reference's hotwords path — the context/hotword preamble with the +// caller's text in the 热词列表 ("hotword list") slot, exactly where +// get_prompt puts its ", ".join(hotwords): +// +// 请结合上下文信息,更加准确地完成语音转写任务。 +// 如果没有相关信息,我们会留空。\n\n\n**上下文信息:**\n\n\n +// 热词列表:[{context}]\n +std::string build_funasr_prompt_text(const char * context, const char * lang, bool use_itn) { std::string out; + if (context != nullptr && context[0] != '\0') { + // 请结合上下文信息,更加准确地完成语音转写任务。 + // 如果没有相关信息,我们会留空。 + out = + "\xE8\xAF\xB7\xE7\xBB\x93\xE5\x90\x88\xE4\xB8\x8A" + "\xE4\xB8\x8B\xE6\x96\x87\xE4\xBF\xA1\xE6\x81\xAF" + "\xEF\xBC\x8C\xE6\x9B\xB4\xE5\x8A\xA0\xE5\x87\x86" + "\xE7\xA1\xAE\xE5\x9C\xB0\xE5\xAE\x8C\xE6\x88\x90" + "\xE8\xAF\xAD\xE9\x9F\xB3\xE8\xBD\xAC\xE5\x86\x99" + "\xE4\xBB\xBB\xE5\x8A\xA1\xE3\x80\x82\xE5\xA6\x82" + "\xE6\x9E\x9C\xE6\xB2\xA1\xE6\x9C\x89\xE7\x9B\xB8" + "\xE5\x85\xB3\xE4\xBF\xA1\xE6\x81\xAF\xEF\xBC\x8C" + "\xE6\x88\x91\xE4\xBB\xAC\xE4\xBC\x9A\xE7\x95\x99" + "\xE7\xA9\xBA\xE3\x80\x82"; + out += "\n\n\n"; + // **上下文信息:** + out += + "\x2A\x2A\xE4\xB8\x8A\xE4\xB8\x8B\xE6\x96\x87\xE4\xBF\xA1" + "\xE6\x81\xAF\xEF\xBC\x9A\x2A\x2A"; + out += "\n\n\n"; + // 热词列表:[ + out += + "\xE7\x83\xAD\xE8\xAF\x8D\xE5\x88\x97\xE8\xA1\xA8" + "\xEF\xBC\x9A\x5B"; + out += context; + out += "]\n"; + } if (lang != nullptr && lang[0] != '\0') { // 语音转写成 = "transcribe to" / "transcribe into" - out = "\xE8\xAF\xAD\xE9\x9F\xB3\xE8\xBD\xAC\xE5\x86\x99\xE6\x88\x90"; + out += "\xE8\xAF\xAD\xE9\x9F\xB3\xE8\xBD\xAC\xE5\x86\x99\xE6\x88\x90"; out += lang; } else { - out = "\xE8\xAF\xAD\xE9\x9F\xB3\xE8\xBD\xAC\xE5\x86\x99"; + out += "\xE8\xAF\xAD\xE9\x9F\xB3\xE8\xBD\xAC\xE5\x86\x99"; } if (!use_itn) { // ,不进行文本规整 = "; do not apply text normalization" @@ -227,6 +261,7 @@ transcribe_status encode_with_chat_specials(const transcribe::Tokenizer & tok, // <|im_start|>/<|im_end|> within each segment. transcribe_status build_funasr_nano_prompt(const transcribe::Tokenizer & tok, const ChatTokens & ct, + const char * context, const char * language, bool use_itn, int fake_token_len, @@ -235,7 +270,19 @@ transcribe_status build_funasr_nano_prompt(const transcribe::Tokenizer & tok, out_ids.clear(); out_fbank_beg = 0; - const std::string prompt_text = build_funasr_prompt_text(language, use_itn); + // Caller context rides inside the user turn that + // encode_with_chat_specials scans for <|im_start|>/<|im_end|> markers; + // reject those literals in context text so caller data cannot inject + // control tokens (mirrors whisper's special-token rejection). + if (context != nullptr && + (std::strstr(context, "<|im_start|>") != nullptr || std::strstr(context, "<|im_end|>") != nullptr)) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "funasr_nano: context text must not contain chat control " + "markers (<|im_start|>/<|im_end|>)"); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + const std::string prompt_text = build_funasr_prompt_text(context, language, use_itn); std::string seg_a = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" @@ -658,7 +705,8 @@ transcribe_status run(transcribe_session * session, std::vector prompt_ids; int fbank_beg = 0; if (const transcribe_status st = - build_funasr_nano_prompt(cm->tok, cm->chat_tokens, lang, use_itn, fake_token_len, prompt_ids, fbank_beg); + build_funasr_nano_prompt(cm->tok, cm->chat_tokens, transcribe_run_params_context(params), lang, use_itn, + fake_token_len, prompt_ids, fbank_beg); st != TRANSCRIBE_OK) { return st; } @@ -1139,8 +1187,8 @@ transcribe_status run_batch(transcribe_session * session, continue; } int fbank_beg = 0; - if (build_funasr_nano_prompt(cm->tok, cm->chat_tokens, lang, use_itn, T_audio[b], prompt_ids[b], fbank_beg) != - TRANSCRIBE_OK) { + if (build_funasr_nano_prompt(cm->tok, cm->chat_tokens, transcribe_run_params_context(params), lang, use_itn, + T_audio[b], prompt_ids[b], fbank_beg) != TRANSCRIBE_OK) { continue; } T_prompt[b] = static_cast(prompt_ids[b].size()); diff --git a/src/arch/qwen3_asr/capabilities.cpp b/src/arch/qwen3_asr/capabilities.cpp index 4737e588..14dc7e58 100644 --- a/src/arch/qwen3_asr/capabilities.cpp +++ b/src/arch/qwen3_asr/capabilities.cpp @@ -20,6 +20,11 @@ void apply_family_invariants(transcribe_model & model) { // Cancellation is wired at the per-run level. No PNC/ITN toggle; the // Whisper-specific features do not apply here. transcribe::set_feature(&model, TRANSCRIBE_FEATURE_CANCELLATION, true); + + // transcribe_run_params::context is honored: the string rides the chat + // template's system slot, Qwen3-ASR's trained biasing channel (the + // reference's TranscribeArgs context / _build_messages system message). + transcribe::set_feature(&model, TRANSCRIBE_FEATURE_INITIAL_PROMPT, true); } } // namespace transcribe::qwen3_asr diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 6e382fbe..06bd3304 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -364,16 +364,21 @@ transcribe_status resolve_chat_tokens(const transcribe::Tokenizer & tok, ChatTok // Build the prompt token sequence + audio-position list, mirroring the // Qwen3-ASR chat template at the token level: // -// <|im_start|>system\n<|im_end|>\n +// <|im_start|>system\n{context}<|im_end|>\n // <|im_start|>user\n<|audio_start|><|audio_pad|>*T_enc<|audio_end|><|im_end|>\n // <|im_start|>assistant\n[language {Name}]? // -// System prompt is empty. A non-null `lang_prefix_ids` (resolved via +// The system slot carries the caller's recognition-biasing context verbatim +// (the reference builds {"role": "system", "content": context or ""} — +// qwen_asr/inference/qwen3_asr.py:_build_messages); a null/empty +// `context_ids` leaves it empty. A non-null `lang_prefix_ids` (resolved via // encode_language_prefix) is appended after the trailing newline to force an -// output language; kept out of here so this stays a pure token-id assembler. +// output language; both are kept out of here so this stays a pure token-id +// assembler. void build_prompt_tokens(const QwenAsrHParams & hp, const ChatTokens & ct, int T_enc, + const std::vector * context_ids, const std::vector * lang_prefix_ids, std::vector & out_ids, std::vector & out_audio_positions) { @@ -383,6 +388,9 @@ void build_prompt_tokens(const QwenAsrHParams & hp, out_ids.push_back(ct.im_start); out_ids.push_back(ct.role_system); out_ids.push_back(ct.newline); + if (context_ids != nullptr && !context_ids->empty()) { + out_ids.insert(out_ids.end(), context_ids->begin(), context_ids->end()); + } out_ids.push_back(ct.im_end); out_ids.push_back(ct.newline); @@ -564,6 +572,19 @@ transcribe_status run(transcribe_session * session, lang_prefix_ptr = &lang_prefix_ids; } + // Recognition-biasing context -> system-message tokens (the reference's + // TranscribeArgs context / _build_messages system slot). Plain BPE; the + // template's control tokens are pushed by build_prompt_tokens itself. + std::vector context_ids; + const std::vector * context_ptr = nullptr; + if (const char * ctx_text = transcribe_run_params_context(params); ctx_text != nullptr) { + if (const transcribe_status st = cm->tok.encode(ctx_text, context_ids); st != TRANSCRIBE_OK) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run: tokenizer.encode failed on context"); + return st; + } + context_ptr = &context_ids; + } + transcribe::debug::init(); // Mel front-end. @@ -725,7 +746,7 @@ transcribe_status run(transcribe_session * session, // Prompt construction. std::vector prompt_ids; std::vector audio_positions; - build_prompt_tokens(cm->hparams, cm->chat_tokens, T_enc, lang_prefix_ptr, prompt_ids, audio_positions); + build_prompt_tokens(cm->hparams, cm->chat_tokens, T_enc, context_ptr, lang_prefix_ptr, prompt_ids, audio_positions); const int T_prompt = static_cast(prompt_ids.size()); const int prefix_len = audio_positions.empty() ? 0 : static_cast(audio_positions.front()); const int suffix_len = T_prompt - prefix_len - T_enc; @@ -1528,6 +1549,18 @@ transcribe_status run_batch(transcribe_session * session, lang_prefix_ptr = &lang_prefix_ids; } + // Shared recognition-biasing context (same one-run_params contract); + // encoded once, injected into every utterance's system slot. + std::vector context_ids; + const std::vector * context_ptr = nullptr; + if (const char * ctx_text = transcribe_run_params_context(params); ctx_text != nullptr) { + if (cm->tok.encode(ctx_text, context_ids) != TRANSCRIBE_OK) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run_batch: tokenizer.encode failed on context"); + return TRANSCRIBE_ERR_GGUF; + } + context_ptr = &context_ids; + } + // Pass 1: per-utterance encoder + prefill into KV slabs. std::vector> generated(n); std::vector T_prompt(n, 0); @@ -1563,7 +1596,7 @@ transcribe_status run_batch(transcribe_session * session, continue; } std::vector ap; - build_prompt_tokens(cm->hparams, cm->chat_tokens, T_enc[b], lang_prefix_ptr, prompt_ids[b], ap); + build_prompt_tokens(cm->hparams, cm->chat_tokens, T_enc[b], context_ptr, lang_prefix_ptr, prompt_ids[b], ap); T_prompt[b] = static_cast(prompt_ids[b].size()); prefix_len = ap.empty() ? 0 : static_cast(ap.front()); // Same gate as single-shot run(); the rest of the batch still runs. diff --git a/src/arch/voxtral/capabilities.cpp b/src/arch/voxtral/capabilities.cpp index 58e69108..e5eded12 100644 --- a/src/arch/voxtral/capabilities.cpp +++ b/src/arch/voxtral/capabilities.cpp @@ -22,10 +22,13 @@ void apply_family_invariants(transcribe_model & model) { // by read_capability_kv at load; default it true here as a fallback. caps.supports_translate = true; - // Per-run cancellation; the free-text prompt is wired via the - // INITIAL_PROMPT feature on the Voxtral run extension. + // Per-run cancellation only. INITIAL_PROMPT stays false: Voxtral's + // transcription template has no recognition-biasing slot for + // transcribe_run_params::context — the only place free text can go is + // instruct mode, which replaces the [TRANSCRIBE] task (a different + // output contract). If free-text instruct is ever exposed, it belongs + // on a voxtral run extension, not the core context field. transcribe::set_feature(&model, TRANSCRIBE_FEATURE_CANCELLATION, true); - transcribe::set_feature(&model, TRANSCRIBE_FEATURE_INITIAL_PROMPT, true); } } // namespace transcribe::voxtral diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index e8c6d711..79cb76e0 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -1476,8 +1476,16 @@ transcribe_status whisper_run(transcribe_session * session, if (prev_sot_id < 0) { prev_sot_id = cm->tok.find("<|startofprev|>"); } - const bool prompt_requested = (wp->prompt_tokens != nullptr && wp->n_prompt_tokens > 0) || - (wp->initial_prompt != nullptr && wp->initial_prompt[0] != '\0'); + // Effective initial-prompt string. The whisper run ext wins when it + // carries one; otherwise the core run_params::context maps onto the same + // previous-context mechanism (transcribe_run_params_context is size-gated + // and returns nullptr for absent/empty, so this is non-null iff some + // caller supplied non-empty prompt text). + const char * initial_prompt_text = (wp->initial_prompt != nullptr && wp->initial_prompt[0] != '\0') ? + wp->initial_prompt : + transcribe_run_params_context(params); + const bool prompt_requested = + (wp->prompt_tokens != nullptr && wp->n_prompt_tokens > 0) || initial_prompt_text != nullptr; if (prev_sot_id < 0 && (prompt_requested || wp->condition_on_prev_tokens)) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "whisper run: model has no <|startofprev|> token; " @@ -1487,9 +1495,10 @@ transcribe_status whisper_run(transcribe_session * session, // Resolve initial prompt -> text-only token ids (library prepends // <|startofprev|>). Two paths: prompt_tokens (caller-owned, verbatim, - // text-side only); or initial_prompt string, tokenized as HF's - // get_prompt_ids form ("<|startofprev|> " + strip) with any special token - // (id >= eos_id) in the text rejected (tokenization_whisper.py). + // text-side only); or the effective prompt string (ext initial_prompt or + // core context), tokenized as HF's get_prompt_ids form + // ("<|startofprev|> " + strip) with any special token (id >= eos_id) in + // the text rejected (tokenization_whisper.py). const int max_prev_cap = wp->max_prev_context_tokens > 0 ? wp->max_prev_context_tokens : (cm->hparams.dec_max_target_positions / 2 - 1); std::vector prompt_text_ids; @@ -1505,8 +1514,8 @@ transcribe_status whisper_run(transcribe_session * session, return TRANSCRIBE_ERR_INVALID_ARG; } prompt_text_ids.assign(wp->prompt_tokens, wp->prompt_tokens + wp->n_prompt_tokens); - } else if (wp->initial_prompt != nullptr && wp->initial_prompt[0] != '\0') { - std::string s(wp->initial_prompt); + } else if (initial_prompt_text != nullptr) { + std::string s(initial_prompt_text); auto issp = [](char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; }; @@ -1540,7 +1549,7 @@ transcribe_status whisper_run(transcribe_session * session, const int id = cm->tok.find(piece); if (id >= eos_id) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "whisper run: initial_prompt contains " + "whisper run: initial_prompt/context contains " "disallowed special token \"%s\" (id %d)", piece.c_str(), id); return TRANSCRIBE_ERR_INVALID_ARG; @@ -1555,13 +1564,13 @@ transcribe_status whisper_run(transcribe_session * session, if (cm->tok.encode(text, prompt_text_ids) != TRANSCRIBE_OK) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "whisper run: tokenizer.encode failed on " - "initial_prompt"); + "initial_prompt/context"); return TRANSCRIBE_ERR_GGUF; } for (int32_t id : prompt_text_ids) { if (id >= eos_id) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "whisper run: initial_prompt contains " + "whisper run: initial_prompt/context contains " "disallowed special token id %d", id); return TRANSCRIBE_ERR_INVALID_ARG; @@ -2576,8 +2585,13 @@ transcribe_status whisper_run_batch(transcribe_session * session, if (prev_sot_id < 0) { prev_sot_id = cm->tok.find("<|startofprev|>"); } - const bool prompt_requested = (wp->prompt_tokens != nullptr && wp->n_prompt_tokens > 0) || - (wp->initial_prompt != nullptr && wp->initial_prompt[0] != '\0'); + // Effective prompt string mirrors the serial path: ext initial_prompt + // wins, core run_params::context is the fallback. + const char * initial_prompt_text = (wp->initial_prompt != nullptr && wp->initial_prompt[0] != '\0') ? + wp->initial_prompt : + transcribe_run_params_context(params); + const bool prompt_requested = + (wp->prompt_tokens != nullptr && wp->n_prompt_tokens > 0) || initial_prompt_text != nullptr; std::vector prev_tokens; if (prompt_requested) { if (prev_sot_id < 0) { @@ -2592,7 +2606,7 @@ transcribe_status whisper_run_batch(transcribe_session * session, } ptext.assign(wp->prompt_tokens, wp->prompt_tokens + wp->n_prompt_tokens); } else { - std::string s(wp->initial_prompt); + std::string s(initial_prompt_text); auto issp = [](char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; }; diff --git a/src/transcribe-session.h b/src/transcribe-session.h index 7d769eaf..c730562b 100644 --- a/src/transcribe-session.h +++ b/src/transcribe-session.h @@ -38,6 +38,25 @@ inline int32_t transcribe_session_params_n_ctx(const struct transcribe_session_p return params->n_ctx; } +// Read transcribe_run_params::context with the same struct_size guard: +// context is a trailing field (appended after spec_k_drafts), so an older +// caller's smaller struct may not include it. Returns nullptr for a NULL +// params, a pre-context struct, or an empty string, so families need only +// one non-null check to mean "caller supplied context text". +inline const char * transcribe_run_params_context(const struct transcribe_run_params * params) { + if (params == nullptr) { + return nullptr; + } + const size_t field_end = offsetof(struct transcribe_run_params, context) + sizeof(params->context); + if (params->struct_size < static_cast(field_end)) { + return nullptr; + } + if (params->context == nullptr || params->context[0] == '\0') { + return nullptr; + } + return params->context; +} + struct transcribe_session { // The model this session was constructed from. Borrowed pointer: // the caller is required (per the public threading contract) to keep @@ -241,6 +260,7 @@ struct transcribe_session { // next begin mutates them. std::string stream_language_owned; std::string stream_target_language_owned; + std::string stream_context_owned; // UI-facing streaming text state. `full_text` above remains the raw // model hypothesis. `stream_committed_text` is the append-only public diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 6e254815..ecd132aa 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -1351,6 +1351,18 @@ void warn_unsupported_advisory(const struct transcribe_model * model, const stru req, arch_name, variant); transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); } + // transcribe_run_params_context is size-gated, so a pre-context caller + // struct reads as "no context" here rather than as garbage bytes. + if (transcribe_run_params_context(rp) != nullptr && + !transcribe::has_feature(model, TRANSCRIBE_FEATURE_INITIAL_PROMPT)) { + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller supplied context text but model '%s' " + "(variant '%s') has no recognition-biasing mechanism; the " + "context is ignored. Use transcribe_model_supports(model, " + "TRANSCRIBE_FEATURE_INITIAL_PROMPT) to pre-check.", + arch_name, variant); + transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); + } } } // namespace @@ -1771,6 +1783,12 @@ static transcribe_status transcribe_stream_begin_impl(struct transcribe_session // wants a run-slot ext at stream begin must plumb it deliberately. session->stream_language_owned = run_params->language != nullptr ? run_params->language : ""; session->stream_target_language_owned = run_params->target_language != nullptr ? run_params->target_language : ""; + // context is size-gated (trailing field): the accessor reads it only + // when the caller's struct is new enough to carry it. + { + const char * ctx_in = transcribe_run_params_context(run_params); + session->stream_context_owned = ctx_in != nullptr ? ctx_in : ""; + } // PREFIX copy, not struct assignment: the size gate above admits any // struct_size >= k_min_run_params_size, so a conforming caller's // allocation may be SHORTER than sizeof (fields past `family`, e.g. @@ -1785,7 +1803,8 @@ static transcribe_status transcribe_stream_begin_impl(struct transcribe_session run_params_owned.language = run_params->language != nullptr ? session->stream_language_owned.c_str() : nullptr; run_params_owned.target_language = run_params->target_language != nullptr ? session->stream_target_language_owned.c_str() : nullptr; - run_params_owned.family = nullptr; + run_params_owned.context = !session->stream_context_owned.empty() ? session->stream_context_owned.c_str() : nullptr; + run_params_owned.family = nullptr; const transcribe_status st = session->model->arch->stream_begin(session, &run_params_owned, stream_params); if (st != TRANSCRIBE_OK) { diff --git a/tests/api_smoke.c b/tests/api_smoke.c index 04ae026b..5abf103c 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -234,6 +234,8 @@ static void test_init_macros(void) { CHECK(rp_macro.target_language == NULL); CHECK(rp_macro.keep_special_tags == false); CHECK(rp_macro.family == NULL); + CHECK(rp_macro.spec_k_drafts == -1); + CHECK(rp_macro.context == NULL); struct transcribe_stream_params sp_macro; transcribe_stream_params_init(&sp_macro); diff --git a/tests/qwen3_asr_smoke.cpp b/tests/qwen3_asr_smoke.cpp index 98ac5ba2..4a61c3b1 100644 --- a/tests/qwen3_asr_smoke.cpp +++ b/tests/qwen3_asr_smoke.cpp @@ -169,6 +169,9 @@ int main() { CHECK_EQ_INT(caps->n_languages, 4); CHECK(caps->languages != nullptr); CHECK_EQ_INT(caps->max_timestamp_kind, TRANSCRIBE_TIMESTAMPS_NONE); + // run_params::context is honored (system-slot biasing) — the + // family invariant advertises it via the feature probe. + CHECK(transcribe_model_supports(model, TRANSCRIBE_FEATURE_INITIAL_PROMPT) == true); } // Internal-view assertions.