From 06391624680a2c29ad8c740eb3f4ad0cf704564c Mon Sep 17 00:00:00 2001 From: jiafatom Date: Wed, 15 Jul 2026 20:27:00 +0000 Subject: [PATCH 1/3] Support chat-style multimodal audio LMs in speech WER evaluation The genai speech evaluator only routed `whisper` and `nemotron_speech` model types and raised for anything else, so chat-style multimodal LMs with an audio input (e.g. gemma4, and other audio-instruction models) could not be evaluated for WER/RTFx without a custom script. Changes: - Detect chat-style audio LMs generically via a `speech`/`audio` component in `genai_config.json` (`_is_multimodal_lm_genai`, not a hard-coded model type) and route them to a new `_inference_text_genai_multimodal` path. It builds the `<|audio|>` chat prompt from the model's `chat_template.jinja` with a configurable system prompt + instruction (defaulting to a strict ASR prompt that suppresses chat-style refusals), greedy-decodes, and returns only the newly generated tokens as the transcription. The system prompt/instruction can be overridden via the metric's pre-process params. Dedicated ASR heads (whisper/nemotron_speech) are unaffected: they use decoder-token prompts and cannot emit chat-style refusals, so a system prompt does not apply. - Normalize text in `WordErrorRate` by default (lowercase, strip punctuation, collapse whitespace), matching standard ASR WER practice. Previously raw text was passed to torchmetrics, so casing/punctuation the model emits but references omit were counted as errors (e.g. FLEURS references are already normalized). Toggleable via `normalize`. Validated end-to-end: gemma4 int4 on FLEURS en_us reports WER 0.0926, matching the reference standalone script (raw/un-normalized was 0.2171). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a5291c2-3155-439c-97cf-4febeab28ac5 --- olive/evaluator/accuracy.py | 26 ++++- olive/evaluator/olive_evaluator.py | 145 ++++++++++++++++++++++++- test/evaluator/test_olive_evaluator.py | 56 ++++++++++ 3 files changed, 224 insertions(+), 3 deletions(-) diff --git a/olive/evaluator/accuracy.py b/olive/evaluator/accuracy.py index 8c443a9217..43d81b832b 100644 --- a/olive/evaluator/accuracy.py +++ b/olive/evaluator/accuracy.py @@ -171,7 +171,24 @@ class WordErrorRate(AccuracyBase): @classmethod def _default_config(cls) -> dict[str, ConfigParam]: - return {} + return { + "normalize": ConfigParam(type_=bool, default_value=True), + } + + @staticmethod + def _normalize(text: str) -> str: + """Normalize text for WER: lowercase, strip punctuation, collapse whitespace. + + Word Error Rate is conventionally reported on normalized text so that casing and + punctuation (which ASR models emit but references often omit) are not counted as + errors. Without this, e.g. FLEURS references (already lowercased, de-punctuated) + compared against a chat-style model's cased/punctuated output roughly doubles the WER. + """ + import re + + text = str(text).lower() + text = re.sub(r"[^\w\s]", " ", text) + return re.sub(r"\s+", " ", text).strip() def measure(self, model_output, target): preds = model_output.preds @@ -186,7 +203,12 @@ def measure(self, model_output, target): elif not isinstance(refs, list): refs = list(refs) - wer = torchmetrics.text.WordErrorRate(**self.config_dict) + config = dict(self.config_dict) + if config.pop("normalize", True): + preds = [self._normalize(p) for p in preds] + refs = [self._normalize(r) for r in refs] + + wer = torchmetrics.text.WordErrorRate(**config) result = wer(preds, refs) return result.item() diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index 75d743602f..3b9edf1a44 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -616,6 +616,20 @@ def _add_array(arr, name): return arrays, names +def _is_multimodal_lm_genai(genai_cfg: Optional[dict]) -> bool: + """Return True for chat-style multimodal LMs that accept an audio input (e.g. gemma4). + + These models are not dedicated ASR heads like whisper/nemotron_speech; they are + instruction-tuned decoders prompted with an ``<|audio|>`` chat turn. They are detected + by the presence of an audio component (``speech``/``audio``) in the genai config's + ``model`` section. + """ + if not genai_cfg: + return False + model_cfg = genai_cfg.get("model", {}) + return any(key in model_cfg for key in ("speech", "audio")) + + def _unwrap_audio_input(input_data): """Strip the speech metadata dict down to the raw audio array(s). @@ -783,10 +797,15 @@ def _evaluate_onnx_accuracy( inference_output, targets = self._inference_text_genai_streaming( model, metric, dataloader, device, execution_providers ) + elif _is_multimodal_lm_genai(genai_cfg): + inference_output, targets = self._inference_text_genai_multimodal( + model, metric, dataloader, device, execution_providers + ) else: raise ValueError( f"Unsupported genai model type '{model_type}' for speech evaluation. " - f"Supported types: 'whisper' (offline), 'nemotron_speech' (streaming). " + f"Supported types: 'whisper' (offline), 'nemotron_speech' (streaming), " + f"and chat-style multimodal LMs with an audio input (e.g. 'gemma4'). " f"For unsupported model types, use a custom evaluation script." ) else: @@ -1259,6 +1278,130 @@ def _transcribe_chunks(audio_arr: np.ndarray, genai_model) -> str: } return OliveModelOutput(preds=all_preds, logits=timing_metadata, extras=all_extras), all_targets + def _inference_text_genai_multimodal( + self, + model: ONNXModelHandler, + metric: Metric, + dataloader: "DataLoader", + device: Device = Device.CPU, + execution_providers: Optional[Union[str, list[str]]] = None, + ) -> tuple[OliveModelOutput, Any]: + """Text-based ASR inference for chat-style multimodal LMs (e.g. gemma4) via onnxruntime-genai. + + Unlike whisper/nemotron_speech, these models are instruction-tuned decoders: the audio is + wrapped in an ``<|audio|>`` chat turn built from the model's ``chat_template.jinja`` and a + configurable system prompt + instruction, then decoded greedily. Only the newly generated + tokens (after the prompt) are returned as the transcription. + + The system prompt and instruction can be overridden via the metric's pre-process params + (``system_prompt`` / ``instruction``); they default to a strict ASR prompt that suppresses + chat-style refusals which would otherwise inflate WER. + """ + try: + import onnxruntime_genai as og + except ImportError: + raise ImportError( + "onnxruntime-genai is required for genai-based speech evaluation. " + "Install it with: pip install onnxruntime-genai" + ) from None + + import io + + import soundfile as sf + + model_dir = _get_genai_model_dir(model) + + with (Path(model_dir) / "genai_config.json").open() as f: + genai_config = json.load(f) + + config = og.Config(model_dir) + config.clear_providers() + if device == Device.GPU: + config.append_provider("cuda") + og_model = og.Model(config) + processor = og_model.create_multimodal_processor() + tokenizer = og.Tokenizer(og_model) + + pre_process_params = ( + metric.data_config.pre_process_data_config.params + if (metric.data_config and metric.data_config.pre_process_data_config) + else {} + ) + sample_rate = pre_process_params.get("sample_rate", 16000) + default_system = ( + "You are an automatic speech recognition (ASR) system. " + "Output only the exact verbatim transcript of the speech in the audio. " + "Do not add commentary, explanations, or apologies. " + "If unsure, output your best guess." + ) + system_prompt = pre_process_params.get("system_prompt", default_system) + instruction = pre_process_params.get("instruction", "Transcribe the audio verbatim.") + max_length = genai_config.get("search", {}).get("max_length", 1024) + + # Build the chat prompt once (audio is injected per-sample via the processor). + chat_template_path = Path(model_dir) / "chat_template.jinja" + template_str = chat_template_path.read_text(encoding="utf-8") if chat_template_path.exists() else None + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": [{"type": "audio"}, {"type": "text", "text": instruction}]}, + ] + prompt = tokenizer.apply_chat_template( + messages=json.dumps(messages), tools="", add_generation_prompt=True, template_str=template_str + ) + + def _transcribe(audio_arr: np.ndarray, genai_model) -> str: + buffer = io.BytesIO() + sf.write(buffer, audio_arr, samplerate=sample_rate, format="WAV") + audios = og.Audios.open_bytes(buffer.getvalue()) + inputs = processor(prompt, audios=audios) + + params = og.GeneratorParams(genai_model) + params.set_search_options( + do_sample=False, max_length=max_length, min_length=0, past_present_share_buffer=False + ) + generator = og.Generator(genai_model, params) + generator.set_inputs(inputs) + + prompt_len = generator.token_count() + while not generator.is_done(): + generator.generate_next_token() + + new_tokens = list(generator.get_sequence(0)[prompt_len:]) + return tokenizer.decode(new_tokens).strip() + + all_preds = [] + all_targets = [] + all_extras = [] + total_audio_duration = 0.0 + total_inference_time = 0.0 + + for batch in dataloader: + input_data, labels = OliveEvaluator.unpack_batch_for_accuracy(batch) + + audio_arrays, audio_names = _normalize_audio_batch(input_data) + if not audio_arrays: + continue + + start_time = time.perf_counter() + for arr, name in zip(audio_arrays, audio_names): + total_audio_duration += len(arr) / sample_rate + all_preds.append(_transcribe(np.asarray(arr, dtype=np.float32), og_model)) + all_extras.append({"audio": name if name is not None else str(len(all_extras))}) + total_inference_time += time.perf_counter() - start_time + + if isinstance(labels, (list, tuple)): + all_targets.extend(labels) + else: + all_targets.append(labels) + + del og_model + + timing_metadata = { + "total_audio_duration": total_audio_duration, + "total_inference_time": total_inference_time, + } + return OliveModelOutput(preds=all_preds, logits=timing_metadata, extras=all_extras), all_targets + def _inference_text_genai_streaming( self, model: ONNXModelHandler, diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index cca73a8fd4..5435ecadbf 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -996,6 +996,62 @@ def test_find_genai_config_ignores_directory(self, tmp_path): assert result is None +class TestIsMultimodalLmGenai: + """Tests for _is_multimodal_lm_genai audio-LM detection.""" + + def test_is_multimodal_lm_genai_true_when_speech_field(self): + """Detect a chat-style audio LM (e.g. gemma4) via the speech component.""" + from olive.evaluator.olive_evaluator import _is_multimodal_lm_genai + + assert _is_multimodal_lm_genai({"model": {"type": "gemma4", "speech": {}}}) is True + + def test_is_multimodal_lm_genai_true_when_audio_field(self): + """Detect an audio LM via an audio component.""" + from olive.evaluator.olive_evaluator import _is_multimodal_lm_genai + + assert _is_multimodal_lm_genai({"model": {"type": "some_lm", "audio": {}}}) is True + + def test_is_multimodal_lm_genai_false_when_no_audio(self): + """A text/vision-only genai config is not an audio LM.""" + from olive.evaluator.olive_evaluator import _is_multimodal_lm_genai + + assert _is_multimodal_lm_genai({"model": {"type": "gemma4", "vision": {}}}) is False + + def test_is_multimodal_lm_genai_false_when_none(self): + """A missing genai config is not an audio LM.""" + from olive.evaluator.olive_evaluator import _is_multimodal_lm_genai + + assert _is_multimodal_lm_genai(None) is False + + +class TestWordErrorRateNormalization: + """Tests for WordErrorRate ASR text normalization.""" + + @staticmethod + def _wer(preds, refs, **config): + from olive.evaluator.accuracy import WordErrorRate + from olive.evaluator.olive_evaluator import OliveModelOutput + + metric = WordErrorRate(config=config or None) + return metric.measure(OliveModelOutput(preds=preds, logits=None), refs) + + def test_measure_ignores_case_and_punctuation_by_default(self): + """Casing and punctuation should not be counted as word errors by default.""" + wer = self._wer(["Hello, World!"], ["hello world"]) + assert wer == 0.0 + + def test_measure_counts_real_word_errors(self): + """A genuine substitution is still counted after normalization.""" + # one substitution out of three reference words -> 1/3 + wer = self._wer(["the quick fox"], ["the slow fox"]) + assert abs(wer - (1.0 / 3.0)) < 1e-6 + + def test_measure_respects_normalize_false(self): + """With normalize disabled, casing/punctuation differences count as errors.""" + wer = self._wer(["Hello, World!"], ["hello world"], normalize=False) + assert wer > 0.0 + + class TestSaveSampleLog: """Tests for OliveEvaluator.save_sample_log.""" From 35c11b8055b2a9b077e9a41f23b0037e0b7d581e Mon Sep 17 00:00:00 2001 From: jiafatom Date: Wed, 15 Jul 2026 20:36:18 +0000 Subject: [PATCH 2/3] Refactor: share genai model load + batch loop across speech eval paths The whisper, streaming, and multimodal-LM speech inference methods each duplicated the onnxruntime-genai import guard, genai_config load + og.Model provider build, and the entire per-batch transcribe/timing/ collect loop (only the per-clip transcribe call differed). Extract two shared helpers on OnnxEvaluator: - `_load_genai_speech_model(model, device)` -> (og, og_model, genai_config, model_dir): import guard, genai_config load, provider selection. - `_run_speech_inference_loop(dataloader, sample_rate, transcribe_fn)`: the shared batch loop, parameterized by a `transcribe_fn(audio) -> str`. Each method now keeps only its model-specific transcribe closure. No behavior change; net -85 lines. Re-validated gemma4 FLEURS en_us (WER 0.0926 unchanged); evaluator test suite 125 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a5291c2-3155-439c-97cf-4febeab28ac5 --- olive/evaluator/olive_evaluator.py | 259 ++++++++++------------------- 1 file changed, 87 insertions(+), 172 deletions(-) diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index 3b9edf1a44..0c9ba3d705 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -1148,19 +1148,13 @@ def _inference_vision_genai( return OliveModelOutput(preds=all_preds, logits=None, extras=all_extras), all_targets - def _inference_text_genai( - self, - model: ONNXModelHandler, - metric: Metric, - dataloader: "DataLoader", - device: Device = Device.CPU, - execution_providers: Optional[Union[str, list[str]]] = None, - ) -> tuple[OliveModelOutput, Any]: - """Text-based inference for speech/ASR metrics using onnxruntime-genai. + def _load_genai_speech_model(self, model: ONNXModelHandler, device: Device): + """Load an ORT GenAI model for speech evaluation. - Auto-detected when the model directory contains genai_config.json. - Uses og.Model with multimodal processor for Whisper-style models. - Automatically chunks audio longer than 30 seconds. + Returns ``(og, og_model, genai_config, model_dir)`` where ``og`` is the imported + ``onnxruntime_genai`` module. Shared by the whisper, streaming, and multimodal-LM + speech inference paths to avoid duplicating the import guard, genai_config load, and + execution-provider selection. """ try: import onnxruntime_genai as og @@ -1170,22 +1164,79 @@ def _inference_text_genai( "Install it with: pip install onnxruntime-genai" ) from None - import io - - import soundfile as sf - model_dir = _get_genai_model_dir(model) - - # Read genai_config to determine model properties with (Path(model_dir) / "genai_config.json").open() as f: genai_config = json.load(f) - # Build og.Model with appropriate execution provider config = og.Config(model_dir) config.clear_providers() if device == Device.GPU: config.append_provider("cuda") og_model = og.Model(config) + return og, og_model, genai_config, model_dir + + @staticmethod + def _run_speech_inference_loop(dataloader, sample_rate, transcribe_fn): + """Run the shared speech-eval batch loop. + + Iterates batches, normalizes audio, times inference, transcribes each clip via + ``transcribe_fn(audio_array) -> str``, and collects predictions, per-sample audio + names, and reference texts. Returns ``(OliveModelOutput, targets)`` with RTFx timing + metadata in ``logits``. Only the per-clip ``transcribe_fn`` differs between the + whisper, streaming, and multimodal-LM speech paths. + """ + all_preds = [] + all_targets = [] + all_extras = [] + total_audio_duration = 0.0 + total_inference_time = 0.0 + + for batch in dataloader: + input_data, labels = OliveEvaluator.unpack_batch_for_accuracy(batch) + + # Convert input to list of audio arrays (with optional file names) + audio_arrays, audio_names = _normalize_audio_batch(input_data) + if not audio_arrays: + continue + + start_time = time.perf_counter() + for arr, name in zip(audio_arrays, audio_names): + total_audio_duration += len(arr) / sample_rate + all_preds.append(transcribe_fn(arr)) + all_extras.append({"audio": name if name is not None else str(len(all_extras))}) + total_inference_time += time.perf_counter() - start_time + + # Collect reference texts + if isinstance(labels, (list, tuple)): + all_targets.extend(labels) + else: + all_targets.append(labels) + + timing_metadata = { + "total_audio_duration": total_audio_duration, + "total_inference_time": total_inference_time, + } + return OliveModelOutput(preds=all_preds, logits=timing_metadata, extras=all_extras), all_targets + + def _inference_text_genai( + self, + model: ONNXModelHandler, + metric: Metric, + dataloader: "DataLoader", + device: Device = Device.CPU, + execution_providers: Optional[Union[str, list[str]]] = None, + ) -> tuple[OliveModelOutput, Any]: + """Text-based inference for speech/ASR metrics using onnxruntime-genai. + + Auto-detected when the model directory contains genai_config.json. + Uses og.Model with multimodal processor for Whisper-style models. + Automatically chunks audio longer than 30 seconds. + """ + import io + + import soundfile as sf + + og, og_model, genai_config, _ = self._load_genai_speech_model(model, device) processor = og_model.create_multimodal_processor() # Determine decoder prompt tokens from model config @@ -1210,7 +1261,7 @@ def _inference_text_genai( prompt = "".join(decoder_prompt_tokens) - def _transcribe_chunks(audio_arr: np.ndarray, genai_model) -> str: + def _transcribe_chunks(audio_arr: np.ndarray) -> str: """Transcribe a single audio array, chunking if longer than 30s.""" if len(audio_arr) <= max_chunk_samples: chunks = [audio_arr] @@ -1227,10 +1278,10 @@ def _transcribe_chunks(audio_arr: np.ndarray, genai_model) -> str: audios = og.Audios.open_bytes(buffer.getvalue()) inputs = processor([prompt], audios=audios) - params = og.GeneratorParams(genai_model) + params = og.GeneratorParams(og_model) params.set_search_options(do_sample=False, max_length=max_length, min_length=0, batch_size=1) - generator = og.Generator(genai_model, params) + generator = og.Generator(og_model, params) generator.set_inputs(inputs) while not generator.is_done(): @@ -1241,42 +1292,7 @@ def _transcribe_chunks(audio_arr: np.ndarray, genai_model) -> str: return " ".join(transcriptions) - all_preds = [] - all_targets = [] - all_extras = [] - total_audio_duration = 0.0 - total_inference_time = 0.0 - - for batch in dataloader: - input_data, labels = OliveEvaluator.unpack_batch_for_accuracy(batch) - - # Convert input to list of audio arrays (with optional file names) - audio_arrays, audio_names = _normalize_audio_batch(input_data) - - if not audio_arrays: - continue - - start_time = time.perf_counter() - for arr, name in zip(audio_arrays, audio_names): - total_audio_duration += len(arr) / sample_rate - transcription = _transcribe_chunks(arr, og_model) - all_preds.append(transcription) - all_extras.append({"audio": name if name is not None else str(len(all_extras))}) - total_inference_time += time.perf_counter() - start_time - - # Collect reference texts - if isinstance(labels, (list, tuple)): - all_targets.extend(labels) - else: - all_targets.append(labels) - - del og_model - - timing_metadata = { - "total_audio_duration": total_audio_duration, - "total_inference_time": total_inference_time, - } - return OliveModelOutput(preds=all_preds, logits=timing_metadata, extras=all_extras), all_targets + return self._run_speech_inference_loop(dataloader, sample_rate, _transcribe_chunks) def _inference_text_genai_multimodal( self, @@ -1297,28 +1313,11 @@ def _inference_text_genai_multimodal( (``system_prompt`` / ``instruction``); they default to a strict ASR prompt that suppresses chat-style refusals which would otherwise inflate WER. """ - try: - import onnxruntime_genai as og - except ImportError: - raise ImportError( - "onnxruntime-genai is required for genai-based speech evaluation. " - "Install it with: pip install onnxruntime-genai" - ) from None - import io import soundfile as sf - model_dir = _get_genai_model_dir(model) - - with (Path(model_dir) / "genai_config.json").open() as f: - genai_config = json.load(f) - - config = og.Config(model_dir) - config.clear_providers() - if device == Device.GPU: - config.append_provider("cuda") - og_model = og.Model(config) + og, og_model, genai_config, model_dir = self._load_genai_speech_model(model, device) processor = og_model.create_multimodal_processor() tokenizer = og.Tokenizer(og_model) @@ -1349,17 +1348,17 @@ def _inference_text_genai_multimodal( messages=json.dumps(messages), tools="", add_generation_prompt=True, template_str=template_str ) - def _transcribe(audio_arr: np.ndarray, genai_model) -> str: + def _transcribe(audio_arr: np.ndarray) -> str: buffer = io.BytesIO() - sf.write(buffer, audio_arr, samplerate=sample_rate, format="WAV") + sf.write(buffer, np.asarray(audio_arr, dtype=np.float32), samplerate=sample_rate, format="WAV") audios = og.Audios.open_bytes(buffer.getvalue()) inputs = processor(prompt, audios=audios) - params = og.GeneratorParams(genai_model) + params = og.GeneratorParams(og_model) params.set_search_options( do_sample=False, max_length=max_length, min_length=0, past_present_share_buffer=False ) - generator = og.Generator(genai_model, params) + generator = og.Generator(og_model, params) generator.set_inputs(inputs) prompt_len = generator.token_count() @@ -1369,38 +1368,7 @@ def _transcribe(audio_arr: np.ndarray, genai_model) -> str: new_tokens = list(generator.get_sequence(0)[prompt_len:]) return tokenizer.decode(new_tokens).strip() - all_preds = [] - all_targets = [] - all_extras = [] - total_audio_duration = 0.0 - total_inference_time = 0.0 - - for batch in dataloader: - input_data, labels = OliveEvaluator.unpack_batch_for_accuracy(batch) - - audio_arrays, audio_names = _normalize_audio_batch(input_data) - if not audio_arrays: - continue - - start_time = time.perf_counter() - for arr, name in zip(audio_arrays, audio_names): - total_audio_duration += len(arr) / sample_rate - all_preds.append(_transcribe(np.asarray(arr, dtype=np.float32), og_model)) - all_extras.append({"audio": name if name is not None else str(len(all_extras))}) - total_inference_time += time.perf_counter() - start_time - - if isinstance(labels, (list, tuple)): - all_targets.extend(labels) - else: - all_targets.append(labels) - - del og_model - - timing_metadata = { - "total_audio_duration": total_audio_duration, - "total_inference_time": total_inference_time, - } - return OliveModelOutput(preds=all_preds, logits=timing_metadata, extras=all_extras), all_targets + return self._run_speech_inference_loop(dataloader, sample_rate, _transcribe) def _inference_text_genai_streaming( self, @@ -1416,40 +1384,22 @@ def _inference_text_genai_streaming( Uses og.StreamingProcessor for stateful chunked inference with silence padding for right-context flushing. """ - try: - import onnxruntime_genai as og - except ImportError: - raise ImportError( - "onnxruntime-genai is required for genai-based speech evaluation. " - "Install it with: pip install onnxruntime-genai" - ) from None - - model_dir = _get_genai_model_dir(model) - - with (Path(model_dir) / "genai_config.json").open() as f: - genai_config = json.load(f) + og, og_model, genai_config, _ = self._load_genai_speech_model(model, device) + tokenizer = og.Tokenizer(og_model) sample_rate = genai_config["model"].get("sample_rate", 16000) chunk_samples = genai_config["model"].get("chunk_samples", 8960) - # Build og.Model with appropriate execution provider - config = og.Config(model_dir) - config.clear_providers() - if device == Device.GPU: - config.append_provider("cuda") - og_model = og.Model(config) - tokenizer = og.Tokenizer(og_model) - # Number of silence chunks for right-context flushing num_silence_chunks = 4 - def _transcribe_streaming(audio_arr: np.ndarray, genai_model) -> str: + def _transcribe_streaming(audio_arr: np.ndarray) -> str: """Transcribe audio using stateful streaming processor.""" audio = audio_arr.astype(np.float32) - stream_processor = og.StreamingProcessor(genai_model) + stream_processor = og.StreamingProcessor(og_model) tokenizer_stream = tokenizer.create_stream() - params = og.GeneratorParams(genai_model) - generator = og.Generator(genai_model, params) + params = og.GeneratorParams(og_model) + generator = og.Generator(og_model, params) transcript = "" @@ -1487,42 +1437,7 @@ def decode_tokens(): return transcript - all_preds = [] - all_targets = [] - all_extras = [] - total_audio_duration = 0.0 - total_inference_time = 0.0 - - for batch in dataloader: - input_data, labels = OliveEvaluator.unpack_batch_for_accuracy(batch) - - # Convert input to list of audio arrays (with optional file names) - audio_arrays, audio_names = _normalize_audio_batch(input_data) - - if not audio_arrays: - continue - - start_time = time.perf_counter() - for arr, name in zip(audio_arrays, audio_names): - total_audio_duration += len(arr) / sample_rate - transcription = _transcribe_streaming(arr, og_model) - all_preds.append(transcription) - all_extras.append({"audio": name if name is not None else str(len(all_extras))}) - total_inference_time += time.perf_counter() - start_time - - # Collect reference texts - if isinstance(labels, (list, tuple)): - all_targets.extend(labels) - else: - all_targets.append(labels) - - del og_model - - timing_metadata = { - "total_audio_duration": total_audio_duration, - "total_inference_time": total_inference_time, - } - return OliveModelOutput(preds=all_preds, logits=timing_metadata, extras=all_extras), all_targets + return self._run_speech_inference_loop(dataloader, sample_rate, _transcribe_streaming) def _evaluate_onnx_latency( self, From a8af8a4fcbc81cea716a9cba2ddce6ebf18bc431 Mon Sep 17 00:00:00 2001 From: jiafatom Date: Wed, 15 Jul 2026 20:40:25 +0000 Subject: [PATCH 3/3] Address review: guard non-dict genai model section; positional chat messages - `_is_multimodal_lm_genai`: return False when the genai config's `model` section is not a dict (e.g. a malformed `"model": null`) instead of raising TypeError on the membership check. Added a regression test. - `apply_chat_template`: pass the messages JSON positionally (it is the first parameter of the ORT-GenAI signature) while keeping the keyword-only options as keywords. gemma4 FLEURS en_us re-validated (WER 0.0926 unchanged). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a5291c2-3155-439c-97cf-4febeab28ac5 --- olive/evaluator/olive_evaluator.py | 4 +++- test/evaluator/test_olive_evaluator.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index 0c9ba3d705..23054e1973 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -627,6 +627,8 @@ def _is_multimodal_lm_genai(genai_cfg: Optional[dict]) -> bool: if not genai_cfg: return False model_cfg = genai_cfg.get("model", {}) + if not isinstance(model_cfg, dict): + return False return any(key in model_cfg for key in ("speech", "audio")) @@ -1345,7 +1347,7 @@ def _inference_text_genai_multimodal( {"role": "user", "content": [{"type": "audio"}, {"type": "text", "text": instruction}]}, ] prompt = tokenizer.apply_chat_template( - messages=json.dumps(messages), tools="", add_generation_prompt=True, template_str=template_str + json.dumps(messages), template_str=template_str, tools="", add_generation_prompt=True ) def _transcribe(audio_arr: np.ndarray) -> str: diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index 5435ecadbf..0c3bc2157b 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -1023,6 +1023,12 @@ def test_is_multimodal_lm_genai_false_when_none(self): assert _is_multimodal_lm_genai(None) is False + def test_is_multimodal_lm_genai_false_when_model_not_dict(self): + """A malformed config with a non-dict model section does not raise.""" + from olive.evaluator.olive_evaluator import _is_multimodal_lm_genai + + assert _is_multimodal_lm_genai({"model": None}) is False + class TestWordErrorRateNormalization: """Tests for WordErrorRate ASR text normalization."""