From 8b2fe63a3d4e7cc331d25f044f45d8f7f643d430 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 21 Aug 2026 18:28:27 +0200 Subject: [PATCH] Add `slug='github'` for GitHub-compatible heading ids `to_html` derives heading ids by Pandoc's rules, which differ from GitHub's in ways that break links written against a GitHub-rendered page. Measured against GitHub's own table-of-contents fragments for 296 public wiki pages, the default rules disagree on 539 of 2210 anchors. Add an opt-in `slug='github'` deriving ids by GitHub's rules instead, for documents that already live on GitHub, where an anchor is a published address. The default is unchanged. Three corners drive the divergence, each confirmed against GitHub's rendering rather than inferred: - Only U+0020 becomes `-`. Other whitespace is dropped, so a heading split across source lines slugs as one word. - Text is neither trimmed nor collapsed, so a heading opening with an image keeps the space it left, and a doubled space gives `--`. - There is no `section` fallback, so an all-digit heading such as a date keeps its digits instead of collapsing to `section`, `section-1`, and so on, which renumber whenever a heading is inserted above them. The character rule uses Unicode general categories (letter, number, mark) rather than a copy of `github-slugger`'s generated regex, which reproduces all 3648 corpus anchors exactly. U+200D is kept because that regex omits it, which keeps emoji sequences intact. End to end, 2285 of 2290 anchors now match GitHub. The 5 remaining are one page whose `## Dustin [@DHowett]` headings hit the separate collision between `[@name]` shortcut reference links and cross-reference syntax; the slug rule is exact given the right heading text. Co-Authored-By: Claude Fable 5 --- docs/DIALECT.md | 4 +++ python/mdhtml/export.py | 10 +++++-- python/mdhtml/md2html.py | 4 ++- src/export_html.rs | 56 +++++++++++++++++++++++++++++++++++----- src/python.rs | 9 +++++-- tests/test_export.py | 34 ++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 11 deletions(-) diff --git a/docs/DIALECT.md b/docs/DIALECT.md index 70f1ea7..aaec99b 100644 --- a/docs/DIALECT.md +++ b/docs/DIALECT.md @@ -90,6 +90,10 @@ Headings use `h1` through `h6`; paragraphs, thematic breaks, and block quotes us Automatic heading ids are an export concern, not part of the parse: `to_mdhtml` emits only authored ids, so identical fragments render identically wherever they later appear. `to_html`'s `auto_ids` option (on by default there) derives an id for each heading without one, and any converter that derives section ids must use the same rules: text is lowercased; whitespace becomes `-`; characters other than letters, numbers, `_`, `-`, and `.` are removed; leading nonletters are removed; and an empty result becomes `section`. Duplicate ids receive `-1`, `-2`, and so on, deduplicated within one export. Explicit ids win and participate in duplicate detection. So `## Hello, world!` exports as `

Hello, world!

`. Embedders rendering several fragments into one page pass `auto_ids=False`, since per-fragment derivation cannot see a sibling fragment's ids. +`to_html`'s `slug='github'` derives those ids by GitHub's rules instead of the rules above. It is for documents that already live on GitHub, where an anchor is a published address: a wiki page's own table of contents, a `README` section link, or any link written against the rendered page. Those links break under the default rules, so the mode trades dialect consistency for keeping them working, and is not the default anywhere. The rules, as `github-slugger` implements them: text is lowercased; characters other than letters, numbers, marks, `_`, `-`, and the space are removed; each remaining space becomes `-`. Duplicates dedupe as above. + +Three corners differ from the default rules and are the reason the mode exists. Only the space U+0020 becomes `-`, so other whitespace is dropped outright and a heading broken across source lines slugs as one word. The text is neither trimmed nor whitespace-collapsed, so a heading opening with an image keeps the space the image left and gains a leading `-`, and a doubled space gives `--`. There is no `section` fallback, so a heading with no slug characters left takes the empty id, and repeats of it dedupe to `-1`, `-2` like any other. Two consequences follow for ordinary prose: a heading ending in a period keeps no period, and a heading that opens with digits keeps them, where the default rules strip to the first letter and fall back to `section` when none remains. + Parse options which infer document structure are off by default. Explicit Markdown syntax remains enabled: for example, an explicit heading id is emitted without any option, and bracket math is recognized because its delimiters state the author's intent. `implicit_figures` enables an inferred transformation. With `frontmatter=True` (the default), a document opening with a `---` line, closed by a `---` or `...` line, whose every non-blank, non-comment line between is `key: value` (at least one), is document metadata rather than content: the parse strips it and returns the pairs as `meta`, with values taken verbatim — no YAML types, one matching pair of surrounding quotes removed. A leading block that doesn't fit this shape is content as usual, so a document starting with a thematic break renders one. Frontmatter never reaches the fragment; consumers decide its rendering (page title, a metadata table) from `meta`. diff --git a/python/mdhtml/export.py b/python/mdhtml/export.py index 992bc3b..7df09d7 100644 --- a/python/mdhtml/export.py +++ b/python/mdhtml/export.py @@ -68,7 +68,8 @@ def _text(el): return " ".join(el.to_text().split()) def to_html(src, dest=None, reftypes: dict | None = None, number_headings=None, hl: str | None = "spans", auto_ids: bool = True, - toc: bool = False, refs: str = "resolve", id_prefix: str = "", fn_salt: str = "", hl_lang=None, code_wrap=None) -> Html: + toc: bool = False, refs: str = "resolve", id_prefix: str = "", fn_salt: str = "", hl_lang=None, code_wrap=None, + slug: str = "pandoc") -> Html: """Lower MDHTML (a string or DocumentFragment; never mutated) to finished HTML: cross-references baked as links, headings and captions numbered, `{=html}` raw data spliced, `colwidths` lowered, and code highlighted. A `div` classed `details` lowers to a `
` element, its @@ -77,6 +78,10 @@ def to_html(src, dest=None, reftypes: dict | None = None, number_headings=None, spaces to hyphens, `-1` suffixes on duplicates); pass `auto_ids=False` when rendering fragments that share a page, where per-fragment derived ids would collide. Authored ids (never auto-derived ones) also get a `data-id` attribute, which anchor displays key on. + `slug='github'` derives those ids by GitHub's rules instead, so anchors match a + GitHub-rendered page and links written against one keep working. It keeps periods out of + the slug, keeps leading digits in, and has no `section` fallback for a heading with no + slug characters, where Pandoc's rules give `section`, `section-1` and so on. `refs='ids'` instead bakes each reference as a working link showing its target id (class `xref`), with no registry, numbering, or failure modes - for live-preview contexts where targets may sit outside the fragment. `refs='lenient'` sits between the two: @@ -91,8 +96,9 @@ def to_html(src, dest=None, reftypes: dict | None = None, number_headings=None, may return replacement markup for the highlighted block (None keeps it; `text` is unescaped). Returns an `Html` str carrying `.warnings`; `dest` also writes it to a file.""" if refs not in ("resolve", "ids", "lenient"): raise ValueError(f"unknown refs mode {refs!r}") + if slug not in ("pandoc", "github"): raise ValueError(f"unknown slug mode {slug!r}") if not isinstance(src, str): src = src.to_html() - out, warnings = _export_html(src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, auto_ids) + out, warnings = _export_html(src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, auto_ids, slug) res = Html(out, warnings) if dest is not None: Path(dest).write_text(res, encoding="utf-8") return res diff --git a/python/mdhtml/md2html.py b/python/mdhtml/md2html.py index 163a4b1..09f7b83 100644 --- a/python/mdhtml/md2html.py +++ b/python/mdhtml/md2html.py @@ -17,6 +17,7 @@ RefsMode = str_enum('RefsMode', 'ids', 'lenient', 'resolve') HlMode = str_enum('HlMode', 'spans', 'api', 'off') NumMode = str_enum('NumMode', 'legal', 'decimal') +SlugMode = str_enum('SlugMode', 'pandoc', 'github') KATEX = "https://cdn.jsdelivr.net/npm/katex@0.16.22/dist" MERMAID = "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs" CACHE = Path.home() / ".cache" / "md2html" @@ -91,6 +92,7 @@ def main( dark_theme: str = "vscode_dark", # Code colors in dark mode templates: bool = True, # Show mustache `{{tokens}}` as styled pills auto_ids: bool = True, # Derive ids for headings + slug: SlugMode = SlugMode.pandoc, # Rule for derived heading ids: the dialect's, or GitHub's, to match anchors on a GitHub-rendered page implicit_figures: bool = True, # Promote image-only paragraphs to figures frontmatter: bool = False, # Recognize leading `key: value` frontmatter: strip it, title the page, prepend a metadata table **kwargs @@ -98,7 +100,7 @@ def main( "Read Markdown and write a finished HTML page" tmpl = dict(templates=MUSTACHE, callbacks={'template_token': mustache_pill}) if templates else {} src = to_mdhtml(read_src(file), implicit_figures=implicit_figures, frontmatter=frontmatter, **tmpl, **kwargs) - html = to_html(src, auto_ids=auto_ids, refs=refs, number_headings=number_headings, toc=toc, hl=None if hl == HlMode.off else hl, code_wrap=_code_wrap) + html = to_html(src, auto_ids=auto_ids, slug=slug, refs=refs, number_headings=number_headings, toc=toc, hl=None if hl == HlMode.off else hl, code_wrap=_code_wrap) for w in [*src.warnings, *html.warnings]: print(w, file=sys.stderr) if src.meta: html = meta_table(src.meta) + html title = src.meta.get("title") or (Path(file).stem if file else "mdhtml") diff --git a/src/export_html.rs b/src/export_html.rs index 7c9bb7d..6ceed3c 100644 --- a/src/export_html.rs +++ b/src/export_html.rs @@ -6,6 +6,7 @@ use std::collections::{HashMap, HashSet}; use fast5ever::{DOCUMENT, Dom, NodeData, NodeId, parse_fragment}; +use unicode_properties::{GeneralCategoryGroup, UnicodeGeneralCategory}; use crate::resolve::{self, HeadingNums, Resolver, target_kind}; @@ -35,6 +36,16 @@ pub enum RefsMode { Lenient, } +/// Which rule `auto_ids` derives heading ids by: `Pandoc` is the dialect's own +/// (see `slug`), `Github` reproduces GitHub's anchors so links written against +/// a GitHub-rendered page keep working (see `slug_github`). +#[derive(PartialEq, Eq, Clone, Copy, Default)] +pub enum SlugMode { + #[default] + Pandoc, + Github, +} + /// Per-code-block hooks. Errors short-circuit the export; the pyo3 bridge /// stores the original Python exception and re-raises it. pub type HlLangHook<'a> = &'a (dyn Fn(&str, Option<&str>) -> Result, String> + Send + Sync); @@ -52,6 +63,7 @@ pub struct HtmlExportOptions<'a> { pub hl_lang: Option>, pub code_wrap: Option>, pub auto_ids: bool, + pub slug: SlugMode, } /// Lower an MDHTML fragment to finished HTML; returns the markup and the @@ -131,6 +143,33 @@ fn slug(text: &str) -> String { if out.is_empty() { "section".to_string() } else { out } } +/// Slug for automatic heading ids, GitHub's derivation rules, as implemented by +/// `github-slugger` and used for GitHub wiki and README anchors. Lowercase, +/// keep letters, numbers and marks along with `-` and `_`, drop everything +/// else, then turn each remaining U+0020 into `-`. +/// +/// Three corners are load-bearing and differ from `slug` above, each confirmed +/// against GitHub's own rendering: +/// +/// - Only U+0020 becomes `-`. Other whitespace, newlines included, is dropped +/// outright, so a heading split across source lines slugs as one word. +/// - The text is neither trimmed nor whitespace-collapsed, so a heading opening +/// with an image gets a leading `-` and a doubled space gives `--`. +/// - There is no `section` fallback: a heading with nothing left is the empty +/// slug, and repeats then dedupe to `-1`, `-2` like any other. +/// +/// U+200D (zero width joiner) survives because `github-slugger`'s character +/// list happens to omit it, which keeps emoji sequences intact. +fn slug_github(text: &str) -> String { + let keep = |ch: char| { + ch == '-' + || ch == '_' + || ch == '\u{200D}' + || matches!(ch.general_category_group(), GeneralCategoryGroup::Letter | GeneralCategoryGroup::Number | GeneralCategoryGroup::Mark) + }; + text.to_lowercase().chars().filter(|&c| c == ' ' || keep(c)).map(|c| if c == ' ' { '-' } else { c }).collect() +} + impl Exporter { fn run(&mut self, opts: &HtmlExportOptions) -> Result<(), String> { self.lower_details(); @@ -147,7 +186,7 @@ impl Exporter { } } if opts.auto_ids { - self.auto_ids(&els); + self.auto_ids(&els, opts); } for &e in &els { let Some(id) = self.dom.attr(e, "id").map(str::to_string) else { @@ -245,17 +284,22 @@ impl Exporter { } } - /// Pandoc-style ids for headings without one: lowercased, spaces to - /// hyphens, punctuation dropped, leading non-letters stripped, `-1` - /// suffixes on duplicates; explicit ids join duplicate detection and win. - fn auto_ids(&mut self, els: &[NodeId]) { + /// Ids for headings without one, by `opts.slug`: Pandoc's rules (lowercased, + /// spaces to hyphens, punctuation dropped, leading non-letters stripped) or + /// GitHub's. Duplicates take a `-1` suffix; explicit ids join duplicate + /// detection and win. GitHub's rules read the heading's text verbatim, + /// since its leading and doubled spaces are significant there. + fn auto_ids(&mut self, els: &[NodeId], opts: &HtmlExportOptions) { let mut taken: HashSet = els.iter().filter_map(|&e| self.dom.attr(e, "id").map(str::to_string)).collect(); for i in 0..self.heads.len() { let h = self.heads[i]; if self.dom.attr(h, "id").is_some() { continue; } - let base = slug(&norm_text(&self.dom, h)); + let base = match opts.slug { + SlugMode::Pandoc => slug(&norm_text(&self.dom, h)), + SlugMode::Github => slug_github(&self.dom.to_text(h)), + }; let mut id = base.clone(); let mut n = 0; while !taken.insert(id.clone()) { diff --git a/src/python.rs b/src/python.rs index 2f1d1f8..64a8b0e 100644 --- a/src/python.rs +++ b/src/python.rs @@ -965,7 +965,7 @@ fn attr_node<'py>(py: Python<'py>, attrs: &Attr) -> PyResult> // --------------------------------------------------------------------------- #[pyfunction] -#[pyo3(signature = (src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, auto_ids))] +#[pyo3(signature = (src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, auto_ids, slug))] fn export_html( py: Python<'_>, src: &str, @@ -979,8 +979,9 @@ fn export_html( hl_lang: Option>, code_wrap: Option>, auto_ids: bool, + slug: &str, ) -> PyResult<(String, Vec)> { - use crate::export_html::{HlMode, HtmlExportOptions, NumberHeadings, RefsMode}; + use crate::export_html::{HlMode, HtmlExportOptions, NumberHeadings, RefsMode, SlugMode}; let number_headings = match number_headings { None => None, Some(o) if o.is_none() => None, @@ -1025,6 +1026,10 @@ fn export_html( hl_lang: hl_lang_c.as_ref().map(|c| c as _), code_wrap: code_wrap_c.as_ref().map(|c| c as _), auto_ids, + slug: match slug { + "github" => SlugMode::Github, + _ => SlugMode::Pandoc, + }, }; let result = if hl_lang.is_none() && code_wrap.is_none() { py.detach(|| crate::export_html::export_html(src, &opts)) diff --git a/tests/test_export.py b/tests/test_export.py index 263330c..0548802 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,3 +1,5 @@ +import re + import pytest from mdhtml import TemplateDelimiter, dialect_css, math_js, parse_mdhtml, to_html, to_md, to_mdhtml @@ -109,6 +111,38 @@ def test_toc(): assert 'Three' in h.split('')[0] # id-less heading still listed +def ids(md, **kw): return re.findall(r']*\bid="([^"]*)"', to_html(to_mdhtml(md), **kw)) + + +def test_slug_github(): + # Each case is a divergence from Pandoc's rules, checked against GitHub's + # own anchor for the same heading. + assert ids('# Footnotes.') == ['footnotes.'] # Pandoc keeps the period + assert ids('# Footnotes.', slug='github') == ['footnotes'] # GitHub drops it + assert ids('# 2021-03-16') == ['section'] # no letter to start from + assert ids('# 2021-03-16', slug='github') == ['2021-03-16'] # digits are kept + assert ids('# --page-file-dir', slug='github') == ['--page-file-dir'] + assert ids('# Using custom.css', slug='github') == ['using-customcss'] + # Only U+0020 becomes '-'; a newline is dropped, so the words run together. + assert ids('# Minutes\n*', slug='github') == ['minutes'] + # Text is neither trimmed nor collapsed: the space left by the image leads. + assert ids('# ![](i.png) Wiki', slug='github') == ['-wiki'] + assert ids('# A B', slug='github') == ['a--b'] + # Emoji go, but the joiner inside a sequence stays, as GitHub's list omits it. + assert ids('# \U0001f477‍♀️ Projects', slug='github') == ['‍️-projects'] + + +def test_slug_github_dedup_and_options(): + assert ids('# Repeat\n\n# Repeat\n\n# Repeat', slug='github') == ['repeat', 'repeat-1', 'repeat-2'] + # An authored id wins and still joins duplicate detection. + assert ids('# Repeat {#repeat}\n\n# Repeat', slug='github') == ['repeat', 'repeat-1'] + # A heading with no slug characters is the empty id, where Pandoc says 'section'. + assert ids('# ***', slug='github') == [''] + assert ids('# ***\n\n# ***', slug='github') == ['', '-1'] + assert ids('# Hello', slug='github', auto_ids=False) == [] # slug mode mints nothing alone + with pytest.raises(ValueError): to_html('

x

', slug='nope') + + def test_api_shape(tmp_path): frag = parse_mdhtml('

Hi

') before = frag.to_html()