From fb3978a05c3b81b780d794b7db46deadb482efbe Mon Sep 17 00:00:00 2001 From: chuenchen309 <48723787+chuenchen309@users.noreply.github.com> Date: Mon, 20 Jul 2026 03:19:39 +0800 Subject: [PATCH] Transliterate HTML entities decoded after the unidecode pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the non-unicode path, HTML entity / decimal / hexadecimal decoding runs after `unidecode()`, and the decoded characters are then only NFKD-normalized. A decoded character that NFKD-decomposes to ASCII survives (the docstring's `Ž -> Ž -> z`), but one that needs transliteration (ß, æ, ©, CJK, ...) is left as raw unicode and stripped by the disallowed-chars pattern, producing an empty/lossy slug: `slugify("ß") == ""` while `slugify("ß") == "ss"`. Run `unidecode()` again after normalizing, mirroring the first normalization block. Idempotent on already-ASCII (entity-free) input. --- slugify/slugify.py | 1 + test.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/slugify/slugify.py b/slugify/slugify.py index 9b5f27f..9daa8df 100644 --- a/slugify/slugify.py +++ b/slugify/slugify.py @@ -151,6 +151,7 @@ def slugify( text = unicodedata.normalize('NFKC', text) else: text = unicodedata.normalize('NFKD', text) + text = unidecode.unidecode(text) # make the text lowercase (optional) if lowercase: diff --git a/test.py b/test.py index fcec4b6..d561851 100644 --- a/test.py +++ b/test.py @@ -166,6 +166,26 @@ def test_html_decimal_on(self): r = slugify(txt, decimal=True) self.assertEqual(r, 'z') + def test_html_decimal_on_transliterated(self): + # A decoded numeric entity must be transliterated to ASCII exactly like + # the same character passed in directly (docstring: Ž -> Ž -> z). + txt = 'ß' # ß + r = slugify(txt, decimal=True) + self.assertEqual(r, slugify('ß')) + self.assertEqual(r, 'ss') + + def test_html_hexadecimal_on_transliterated(self): + txt = 'ß' # ß + r = slugify(txt, hexadecimal=True) + self.assertEqual(r, slugify('ß')) + self.assertEqual(r, 'ss') + + def test_html_entities_on_transliterated(self): + txt = 'ß' # ß + r = slugify(txt, entities=True) + self.assertEqual(r, slugify('ß')) + self.assertEqual(r, 'ss') + def test_html_decimal_off(self): txt = 'Ž' r = slugify(txt, entities=False, decimal=False)