diff --git a/src/contextfit/cli.py b/src/contextfit/cli.py index 4b4329c..31ee425 100644 --- a/src/contextfit/cli.py +++ b/src/contextfit/cli.py @@ -458,6 +458,24 @@ def _extract_email_text(raw: str, path: Path) -> tuple[str, dict]: return text, {"source": str(path)} +def _looks_like_email(path: Path, raw: str) -> bool: + """True only for real emails: .eml files, markdown-exported emails, or files + that open with RFC-5322 headers. + + Everything else must NOT be routed through MIME extraction: compat32 + ``get_payload(decode=True)`` on a str payload round-trips the text through + raw-unicode-escape + utf-8/ignore, which silently deletes every char in + U+0080..U+00FF (umlauts, ß, accents) and turns chars above U+00FF into + literal ``\\uXXXX`` sequences. + """ + return ( + path.suffix == ".eml" + or bool(re.match(r"#\s+Email:", raw.lstrip())) + or "**From:**" in raw[:500] + or bool(re.match(r"(From|Return-Path|Received|Delivered-To):", raw)) + ) + + def _discover_files(source: Path) -> list[Path]: if source.is_file(): return [source] @@ -481,8 +499,11 @@ def _preprocess_file( if max_file_bytes and size > max_file_bytes: return {"path": str(path), "status": "skipped", "reason": f"file_too_large:{size}", "bytes": size, "seconds": time.time() - started} - raw = path.read_text(errors="ignore") - text, email_meta = _extract_email_text(raw, path) + raw = path.read_text(encoding="utf-8", errors="ignore") + if _looks_like_email(path, raw): + text, email_meta = _extract_email_text(raw, path) + else: + text, email_meta = raw, {"source": str(path)} tokenizer = Tokenizer.load(tokenizer_name) # Structure-aware pre-chunking by file type. The tokenizer remains the diff --git a/tests/test_ingest_unicode.py b/tests/test_ingest_unicode.py new file mode 100644 index 0000000..4bc0617 --- /dev/null +++ b/tests/test_ingest_unicode.py @@ -0,0 +1,45 @@ +from contextfit.cli import _looks_like_email, _preprocess_file +from contextfit.core.tokenizer import Tokenizer + +SAMPLE_MD = ( + "# Notizen Köln\n" + "\n" + "Grüße aus der Straße: ökonomisch, äußerst übermäßig — ß bleibt ß.\n" + "Beyond Latin-1: 日本語のテキスト und emoji ✅.\n" +) + + +def test_plain_markdown_is_not_treated_as_email(tmp_path): + path = tmp_path / "note.md" + path.write_text(SAMPLE_MD, encoding="utf-8") + assert not _looks_like_email(path, path.read_text(encoding="utf-8")) + + +def test_eml_and_markdown_email_are_treated_as_email(tmp_path): + assert _looks_like_email(tmp_path / "mail.eml", "From: a@b.c\n\nbody\n") + assert _looks_like_email( + tmp_path / "mail.md", "# Email: Update\n**From:** a@b.c\n\n## Content\nhi\n" + ) + assert _looks_like_email( + tmp_path / "raw.txt", "From: Alice \nSubject: hi\n\nbody\n" + ) + + +def test_markdown_ingest_preserves_non_ascii(tmp_path): + path = tmp_path / "note.md" + path.write_text(SAMPLE_MD, encoding="utf-8") + + result = _preprocess_file( + path, + tokenizer_name="cl100k_base", + chunk_size=64, + overlap=0, + max_file_bytes=0, + max_file_tokens=0, + ) + + assert result["status"] == "ok" + tokenizer = Tokenizer.load("cl100k_base") + text = "\n".join(tokenizer.decode(item["tokens"]) for item in result["token_items"]) + for needle in ("Köln", "Grüße", "Straße", "äußerst", "日本語", "✅"): + assert needle in text