From 1e217e6b9cf22246d0ed1b8f249faa57d9667f40 Mon Sep 17 00:00:00 2001 From: Akaash Mohan Saxena Date: Sun, 16 Aug 2026 14:12:22 -0400 Subject: [PATCH] fix(perplexity): use all_special_tokens for GPT-2 tokenizer compatibility GPT-2's slow tokenizer (GPT2Tokenizer) does not expose special_tokens_map_extended, causing an AttributeError when computing perplexity with batch_size > 1. Additionally, the pad_token guard was conditioned on batch_size > 1, but the tokenizer is always called with padding=True, meaning pad_token is required even for batch_size=1 when multiple predictions are passed. Fix: - Replace special_tokens_map_extended.values() with all_special_tokens, which is defined on PreTrainedTokenizerBase and returns a flat list of strings for both slow and fast tokenizers. - Remove the batch_size > 1 condition so pad_token is set whenever it is missing, matching the actual padding requirement. Fixes #766 --- metrics/perplexity/perplexity.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/metrics/perplexity/perplexity.py b/metrics/perplexity/perplexity.py index 557172cd..0ec27098 100644 --- a/metrics/perplexity/perplexity.py +++ b/metrics/perplexity/perplexity.py @@ -116,15 +116,18 @@ def _compute( tokenizer = AutoTokenizer.from_pretrained(model_id) - # if batch_size > 1 (which generally leads to padding being required), and - # if there is not an already assigned pad_token, assign an existing - # special token to also be the padding token - if tokenizer.pad_token is None and batch_size > 1: - existing_special_tokens = list(tokenizer.special_tokens_map_extended.values()) + # Padding is always required when tokenizing multiple predictions together + # (tokenizer is called with padding=True regardless of batch_size). + # Use all_special_tokens instead of special_tokens_map_extended.values() for + # compatibility with both slow and fast tokenizers (e.g. GPT-2's slow tokenizer + # does not expose special_tokens_map_extended). See: + # https://github.com/huggingface/evaluate/issues/766 + if tokenizer.pad_token is None: + existing_special_tokens = tokenizer.all_special_tokens # check that the model already has at least one special token defined assert ( len(existing_special_tokens) > 0 - ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1." + ), "Model must have at least one special token to use for padding. Please use a different model or set batch_size=1." # assign one of the special tokens to also be the pad token tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]})