From fe4ad7ea90097f133f80bddbdb6372d3c7ce9102 Mon Sep 17 00:00:00 2001 From: Oliver Wrede Date: Thu, 2 Jul 2026 22:56:42 +0200 Subject: [PATCH] ingest: stop routing plain .md/.txt through MIME extraction (fixes #1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _preprocess_file sent every discovered file through _extract_email_text. For anything that is not a markdown-exported email, that takes the raw MIME path: compat32 get_payload(decode=True) on a str payload round-trips the text through raw-unicode-escape + utf-8/ignore, silently deleting all chars in U+0080..U+00FF (umlauts, ß, accents) and turning chars above U+00FF into literal \uXXXX sequences. A German-language vault loses every umlaut in the index ('Köln' -> 'Kln'). Gate MIME extraction behind _looks_like_email(): only .eml files, markdown-exported emails (# Email: / **From:**), or files opening with RFC-5322 headers are parsed as email; everything else is ingested verbatim. Also read files as explicit UTF-8 instead of the locale default. Regression test ingests a .md with umlauts/CJK/emoji end-to-end and asserts the decoded token stream still contains them. Co-Authored-By: Claude Fable 5 --- src/contextfit/cli.py | 25 ++++++++++++++++++-- tests/test_ingest_unicode.py | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tests/test_ingest_unicode.py diff --git a/src/contextfit/cli.py b/src/contextfit/cli.py index 4b4329c..31ee425 100644 --- a/src/contextfit/cli.py +++ b/src/contextfit/cli.py @@ -458,6 +458,24 @@ def _extract_email_text(raw: str, path: Path) -> tuple[str, dict]: return text, {"source": str(path)} +def _looks_like_email(path: Path, raw: str) -> bool: + """True only for real emails: .eml files, markdown-exported emails, or files + that open with RFC-5322 headers. + + Everything else must NOT be routed through MIME extraction: compat32 + ``get_payload(decode=True)`` on a str payload round-trips the text through + raw-unicode-escape + utf-8/ignore, which silently deletes every char in + U+0080..U+00FF (umlauts, ß, accents) and turns chars above U+00FF into + literal ``\\uXXXX`` sequences. + """ + return ( + path.suffix == ".eml" + or bool(re.match(r"#\s+Email:", raw.lstrip())) + or "**From:**" in raw[:500] + or bool(re.match(r"(From|Return-Path|Received|Delivered-To):", raw)) + ) + + def _discover_files(source: Path) -> list[Path]: if source.is_file(): return [source] @@ -481,8 +499,11 @@ def _preprocess_file( if max_file_bytes and size > max_file_bytes: return {"path": str(path), "status": "skipped", "reason": f"file_too_large:{size}", "bytes": size, "seconds": time.time() - started} - raw = path.read_text(errors="ignore") - text, email_meta = _extract_email_text(raw, path) + raw = path.read_text(encoding="utf-8", errors="ignore") + if _looks_like_email(path, raw): + text, email_meta = _extract_email_text(raw, path) + else: + text, email_meta = raw, {"source": str(path)} tokenizer = Tokenizer.load(tokenizer_name) # Structure-aware pre-chunking by file type. The tokenizer remains the diff --git a/tests/test_ingest_unicode.py b/tests/test_ingest_unicode.py new file mode 100644 index 0000000..4bc0617 --- /dev/null +++ b/tests/test_ingest_unicode.py @@ -0,0 +1,45 @@ +from contextfit.cli import _looks_like_email, _preprocess_file +from contextfit.core.tokenizer import Tokenizer + +SAMPLE_MD = ( + "# Notizen Köln\n" + "\n" + "Grüße aus der Straße: ökonomisch, äußerst übermäßig — ß bleibt ß.\n" + "Beyond Latin-1: 日本語のテキスト und emoji ✅.\n" +) + + +def test_plain_markdown_is_not_treated_as_email(tmp_path): + path = tmp_path / "note.md" + path.write_text(SAMPLE_MD, encoding="utf-8") + assert not _looks_like_email(path, path.read_text(encoding="utf-8")) + + +def test_eml_and_markdown_email_are_treated_as_email(tmp_path): + assert _looks_like_email(tmp_path / "mail.eml", "From: a@b.c\n\nbody\n") + assert _looks_like_email( + tmp_path / "mail.md", "# Email: Update\n**From:** a@b.c\n\n## Content\nhi\n" + ) + assert _looks_like_email( + tmp_path / "raw.txt", "From: Alice \nSubject: hi\n\nbody\n" + ) + + +def test_markdown_ingest_preserves_non_ascii(tmp_path): + path = tmp_path / "note.md" + path.write_text(SAMPLE_MD, encoding="utf-8") + + result = _preprocess_file( + path, + tokenizer_name="cl100k_base", + chunk_size=64, + overlap=0, + max_file_bytes=0, + max_file_tokens=0, + ) + + assert result["status"] == "ok" + tokenizer = Tokenizer.load("cl100k_base") + text = "\n".join(tokenizer.decode(item["tokens"]) for item in result["token_items"]) + for needle in ("Köln", "Grüße", "Straße", "äußerst", "日本語", "✅"): + assert needle in text