diff --git a/great_docs/assets/post-render.py b/great_docs/assets/post-render.py index 16efaa2c..1d076603 100644 --- a/great_docs/assets/post-render.py +++ b/great_docs/assets/post-render.py @@ -1802,153 +1802,6 @@ def _replace_toc(m): return html_content -def fix_doctest_blockquotes(html_content): - """ - Convert nested blockquotes from doctest `>>>` lines into code blocks. - - When the renderer produces an Example section with raw `>>>` lines in the `.qmd` file, - Quarto/Pandoc interprets the leading `>` characters as Markdown blockquote markers. A `>>>` - line becomes triple-nested `
` elements: - - ```html --- ``` - - This function detects that pattern inside Example/Examples doc-sections and replaces it with a - proper `----func(x) 'result'
-` block so the content renders in monospace as code. - """ - # Match one or more triple-nested blockquote clusters inside an - # Example or Examples doc-section. - _SECTION_RE = re.compile( - r'(]*class="[^"]*doc-section-examples?[^"]*"[^>]*>\s*' - r" ]*>.*?
\s*)" - r"((?:\s*]*>\s*\s*)+)", - re.DOTALL, - ) - - # Extract individual]*>\s*\s*]*>" - r"\s*\s*.*?
\s*content from triple-nested blockquotes - _BQ_TEXT_RE = re.compile( - r"
]*>\s*", - re.DOTALL, - ) - - def _replace_section(m): - header = m.group(1) - bq_block = m.group(2) - lines = [] - for bq in _BQ_TEXT_RE.finditer(bq_block): - text = bq.group(1).strip() - # Reconstruct the doctest line with >>> prefix - lines.append(f">>> {text}") - code = "\n".join(lines) - return f"{header}]*>\s*\s*]*>" - r"\s*\s*(.*?)
\s*\n" - - return _SECTION_RE.sub(_replace_section, html_content) - - -def fix_plain_doctest_code_blocks(html_content): - """ - Convert plain `{code}` blocks containing doctest `>>>` lines into properly highlighted - Python code blocks. - - When the renderer renders consecutive doctest examples separated by blank lines, only the first - block gets a proper ```` ```python ```` fence. Subsequent blocks become 4-space-indented text in - the `.qmd` file, which Quarto renders as plain ```` without syntax highlighting. - - This function finds those plain code blocks, re-highlights them with Pygments, and wraps them in - the same `sourceCode python` structure that Quarto uses for fenced code blocks. - """ - # Matchblocks that contain >>> (i.e. >>>) but - # are NOT already inside a sourceCode div. We look for- # (no class) immediately, which distinguishes them from Quarto's - #blocks. - _PLAIN_DOCTEST_RE = re.compile( - r"", - re.DOTALL, - ) - - # Track a counter for generating unique cb IDs - _cb_counter = [0] - - def _find_max_cb_id(html): - """Find the highest existing cb ID to avoid collisions.""" - ids = re.findall(r'id="cb(\d+)"', html) - return max(int(i) for i in ids) if ids else 0 - - _cb_counter[0] = _find_max_cb_id(html_content) - - def _replace_plain_doctest(m): - code_html = m.group(1) - - # Only process blocks that contain doctest >>> markers - if ">>>" not in code_html: - return m.group(0) - - # Decode HTML entities to get plain text for Pygments - plain_text = code_html - plain_text = plain_text.replace("<", "<") - plain_text = plain_text.replace(">", ">") - plain_text = plain_text.replace("&", "&") - plain_text = plain_text.replace(""", '"') - plain_text = plain_text.replace("'", "'") - # Strip any existing HTML tags (unlikely but safe) - plain_text = re.sub(r"<[^>]+>", "", plain_text) - - # Highlight with Pygments - lexer = PythonLexer() - formatter = HtmlFormatter(nowrap=True, classprefix="") - highlighted = highlight(plain_text, lexer, formatter) - - # Map Pygments CSS classes to Quarto CSS classes - for pg_class, quarto_class in PYGMENTS_TO_QUARTO_CLASS.items(): - if quarto_class: - highlighted = highlighted.replace(f'class="{pg_class}"', f'class="{quarto_class}"') - else: - highlighted = re.sub( - rf'([^<]*)', - r"\1", - highlighted, - ) - - # Assign a unique cb ID - _cb_counter[0] += 1 - cb_id = f"cb{_cb_counter[0]}" - - # Wrap each line in a span with proper id for line linking - lines = highlighted.rstrip("\n").split("\n") - wrapped_lines = [] - for j, line in enumerate(lines, 1): - span_id = f"{cb_id}-{j}" - wrapped_lines.append( - f'' - f'' - f"{line}" - ) - highlighted = "\n".join(wrapped_lines) - - return ( - f'(.*?)' - f'" - f'" - ) - - return _PLAIN_DOCTEST_RE.sub(_replace_plain_doctest, html_content) - - def translate_rst_references(html_content): """ Convert RST citation references into a styled numbered list. @@ -2104,13 +1957,6 @@ def fix_dataclass_attributes(content_str): # Translate renderer-rendered headings and Usage/Source labels content = translate_renderer_headings(content) - # Fix doctest >>> lines that Quarto rendered as nested blockquotes - content = fix_doctest_blockquotes(content) - - # Fix plain' - f'' - f'' - f"{highlighted}" - f"blocks containing >>> doctest lines - # (consecutive examples where only the first got a proper code fence) - content = fix_plain_doctest_code_blocks(content) - # Translate RST citation references (.. [1] ...) content = translate_rst_references(content) diff --git a/tests/renderer/test_doctest_normalization.py b/tests/renderer/test_doctest_normalization.py index 1356e8f5..f3ad7d88 100644 --- a/tests/renderer/test_doctest_normalization.py +++ b/tests/renderer/test_doctest_normalization.py @@ -1,7 +1,11 @@ +import textwrap + import griffe as gf import pytest +from great_docs._apiref._tools import _render from great_docs._builtin.normalization._doctest import normalize_doctests +from great_docs.hooks._object_resolved import emit_object_resolved def _function(text: str, parser: str) -> gf.Function: @@ -52,3 +56,241 @@ def test_doctest_normalization_registers_on_import(): from great_docs.hooks import _object_resolved assert normalize_doctests in _object_resolved.REGISTRY + + +def _render_with_hooks(source: str, name: str, parser: str = "numpy") -> str: + """Render the named object of a source snippet to qmd, with the hooks applied""" + with gf.temporary_visited_package( + "package", {"__init__.py": textwrap.dedent(source)}, docstring_parser=parser + ) as package: + obj = emit_object_resolved(package[name]) + assert obj is not None + for member in obj.members.values(): + _ = emit_object_resolved(member) + return _render(obj) + + +def _unfenced_prompts(qmd: str) -> list[str]: + """Return the doctest prompt lines of `qmd` that fall outside a code fence""" + outside: list[str] = [] + in_fence = False + for line in qmd.split("\n"): + stripped = line.lstrip() + if stripped.startswith("```"): + in_fence = not in_fence + elif stripped.startswith(">>>") and not in_fence: + outside.append(line) + return outside + + +@pytest.mark.parametrize( + ("name", "source"), + [ + ( + "prose_then_code", + ''' + def prose_then_code(): + """ + Do a thing. + + Examples + -------- + Some explanatory prose. + + >>> prose_then_code() + 3 + """ + ''', + ), + ( + "two_groups", + ''' + def two_groups(): + """ + Do two things. + + Examples + -------- + >>> two_groups() + 1 + + >>> two_groups() + 2 + """ + ''', + ), + ( + "no_section", + ''' + def no_section(): + """ + Do an unsectioned thing. + + >>> no_section() + 3 + """ + ''', + ), + ( + "google_header", + ''' + def google_header(): + """ + Do a thing. + + Example: + >>> google_header() + 3 + """ + ''', + ), + ( + "rst_literal", + ''' + def rst_literal(): + """ + Do a thing. + + Examples + -------- + :: + + >>> rst_literal() + 3 + """ + ''', + ), + ( + "indented_groups", + ''' + def indented_groups(): + """ + Do two things. + + Examples + -------- + Indented block: + + >>> indented_groups() + 1 + + >>> indented_groups() + 2 + """ + ''', + ), + ( + "nested_in_list", + ''' + def nested_in_list(): + """ + Do a listed thing. + + Examples + -------- + - First bullet: + + >>> nested_in_list() + 1 + """ + ''', + ), + ( + "Widget", + ''' + class Widget: + """ + A widget. + + Examples + -------- + >>> Widget() ++ """ + + def resize(self): + """ + Resize the widget. + + Examples + -------- + >>> for i in range(2): + ... print(i) + 0 + 1 + """ + ''', + ), + ], +) +def test_every_doctest_prompt_reaches_the_qmd_fenced(name: str, source: str): + """ + No doctest prompt survives into the qmd outside a code fence + + An unfenced prompt reaches Quarto as markdown, where a leading `>` is a + blockquote marker and the block is left unhighlighted. The prompts must be + fenced whatever their shape: inside an `Examples` section or loose in the + subject, one group or several, indented under a literal block, a paragraph + or a list item, and on a class or on its members. + """ + qmd = _render_with_hooks(source, name) + + assert ">>>" in qmd, "the sample rendered without its doctest at all" + assert _unfenced_prompts(qmd) == [] + + +@pytest.mark.parametrize( + ("parser", "docstring"), + [ + ( + "google", + """ + Do a thing. + + Args: + value: The value. + + Examples: + >>> convert(1) + 1 + + >>> convert(2) + 2 + """, + ), + ( + "sphinx", + """ + Do a thing. + + :param value: The value. + + .. rubric:: Examples + + >>> convert(1) + 1 + + >>> convert(2) + 2 + """, + ), + ], +) +def test_native_dialect_doctests_reach_the_qmd_fenced(parser: str, docstring: str): + """ + A docstring written in its configured dialect keeps its prompts fenced + + The dialect decides how griffe splits the sections, so each one reaches the + fencing hook differently. + """ + source = f''' + def convert(value): + """ + {textwrap.indent(textwrap.dedent(docstring), " " * 8).strip()} + """ + ''' + + qmd = _render_with_hooks(source, "convert", parser=parser) + + assert ">>>" in qmd, "the sample rendered without its doctest at all" + assert _unfenced_prompts(qmd) == [] diff --git a/tests/test_post_render.py b/tests/test_post_render.py index 5f92f5b9..b3eee978 100644 --- a/tests/test_post_render.py +++ b/tests/test_post_render.py @@ -28,10 +28,6 @@ def _get_functions(): source = _SCRIPT.read_text() - from pygments import highlight as _highlight - from pygments.formatters import HtmlFormatter as _HtmlFormatter - from pygments.lexers import PythonLexer as _PythonLexer - # Stub _t so translated labels fall back to English def _t(key: str, fallback: str | None = None) -> str: return fallback if fallback is not None else key @@ -42,23 +38,12 @@ def _t(key: str, fallback: str | None = None) -> str: "os": _os, "re": _re, "__builtins__": __builtins__, - "highlight": _highlight, - "HtmlFormatter": _HtmlFormatter, - "PythonLexer": _PythonLexer, "_t": _t, } - # Extract PYGMENTS_TO_QUARTO_CLASS dict (needed by fix_plain_doctest_code_blocks) - cm_start = source.find("PYGMENTS_TO_QUARTO_CLASS = {") - if cm_start != -1: - cm_rest = source[cm_start:] - cm_end = cm_rest.find("}\n") + 2 - exec(cm_rest[:cm_end], ns) - # Extract function definitions by finding their source blocks funcs_to_extract = [ "translate_sphinx_roles", - "fix_plain_doctest_code_blocks", "_postprocess_markdown_content", ] @@ -87,14 +72,12 @@ def _t(key: str, fallback: str | None = None) -> str: return ( ns["translate_sphinx_roles"], - ns["fix_plain_doctest_code_blocks"], ns["_postprocess_markdown_content"], ) ( translate_sphinx_roles, - fix_plain_doctest_code_blocks, postprocess_markdown_content, ) = _get_functions() @@ -277,69 +260,6 @@ def test_type_role(self): assert result == " Is
" -class TestFixPlainDoctestCodeBlocks: - """Tests for fix_plain_doctest_code_blocks (site 137 regression).""" - - def test_single_plain_doctest_converted(self): - """A plainint.block with >>> gets proper sourceCode styling.""" - html = "" - result = fix_plain_doctest_code_blocks(html) - assert 'class="sourceCode python' in result - assert 'class="code-copy-outer-scaffold"' in result - assert ">>> foo(\"hello\")\n'world'" not in result - - def test_consecutive_plain_doctests_all_converted(self): - """Multiple consecutive plain doctest blocks are all converted.""" - html = ( - # First block (already styled by Quarto — should be left alone) - '' - '\n' - # Second block (plain — should be converted) - '' - '' - '>>> schedule("cleanup")\n' - 'True\n" - # Third block (plain — should be converted) - '>>> schedule("backup", delay=60.0)\n' - "True" - ) - result = fix_plain_doctest_code_blocks(html) - # The already-styled block should remain - assert result.count('class="sourceCode python') >= 3 - # No plain>>> schedule("cleanup")\n' - "Falseblocks should remain - assert "" not in result - - def test_plain_code_block_without_doctest_unchanged(self): - """A plainwithout >>> is left as-is.""" - html = "" - result = fix_plain_doctest_code_blocks(html) - assert result == html - - def test_unique_cb_ids_no_collision(self): - """Generated cb IDs don't collide with existing ones.""" - html = ( - 'just some plain text' - '\n" - "' - "existing\n" - ">>> first()\n1" - ) - result = fix_plain_doctest_code_blocks(html) - # Existing cb3 plus two new blocks should give cb4 and cb5 - assert 'id="cb4"' in result - assert 'id="cb5"' in result - - def test_html_entities_decoded_for_highlighting(self): - """HTML entities in the plain block are decoded before Pygments.""" - html = ">>> second()\n2" - result = fix_plain_doctest_code_blocks(html) - # Should be in a sourceCode block, not plain - assert 'class="sourceCode python' in result - # The original plain block should be gone - assert ">>> x < 10 & y > 5\nTrue" not in result - - def _make_autolink(inventory): """Create an autolink function bound to a given inventory.""" import re as _re @@ -797,44 +717,6 @@ def test_multiple_codes_in_one_page(self): assert 'href="../reference/execute.html#pkg.execute"' in result -class TestFixPlainDoctestGdCodeNav: - """Tests that fix_plain_doctest_code_blocks emits the gd-code-nav copy button.""" - - def test_converted_block_has_gd_code_nav(self): - """Converted doctest block should contain a gd-code-nav element.""" - html = "" - result = fix_plain_doctest_code_blocks(html) - assert 'class="gd-code-nav"' in result - - def test_converted_block_has_gd_code_copy_button(self): - """Converted block should have a gd-code-copy button inside the nav.""" - html = ">>> foo()\n42" - result = fix_plain_doctest_code_blocks(html) - assert 'class="gd-code-copy"' in result - assert 'title="Copy to clipboard"' in result - - def test_no_legacy_code_copy_button(self): - """Converted block should NOT have the old code-copy-button class.""" - html = ">>> bar(1, 2)\n3" - result = fix_plain_doctest_code_blocks(html) - assert "code-copy-button" not in result - assert '' not in result - - def test_nav_is_inside_scaffold(self): - """gd-code-nav should be nested inside code-copy-outer-scaffold.""" - html = ">>> baz()\nNone" - result = fix_plain_doctest_code_blocks(html) - scaffold_start = result.find('class="code-copy-outer-scaffold"') - nav_start = result.find('class="gd-code-nav"') - assert scaffold_start < nav_start - - def test_no_code_with_copy_class_on_pre(self): - """Converted>>> x = 1\nshould NOT have the Quarto 'code-with-copy' class.""" - html = "" - result = fix_plain_doctest_code_blocks(html) - assert "code-with-copy" not in result - - # --------------------------------------------------------------------------- # fix_script_paths — back-to-top.js path resolution # --------------------------------------------------------------------------->>> hello()\n'world'