ingest: stop routing plain .md/.txt through MIME extraction (fixes #1)

src/contextfit/cli.py

@@ -458,6 +458,24 @@ def _extract_email_text(raw: str, path: Path) -> tuple[str, dict]:
         return text, {"source": str(path)}
 
 
+def _looks_like_email(path: Path, raw: str) -> bool:
+    """True only for real emails: .eml files, markdown-exported emails, or files
+    that open with RFC-5322 headers.
+
+    Everything else must NOT be routed through MIME extraction: compat32
+    ``get_payload(decode=True)`` on a str payload round-trips the text through
+    raw-unicode-escape + utf-8/ignore, which silently deletes every char in
+    U+0080..U+00FF (umlauts, ß, accents) and turns chars above U+00FF into
+    literal ``\\uXXXX`` sequences.
+    """
+    return (
+        path.suffix == ".eml"
+        or bool(re.match(r"#\s+Email:", raw.lstrip()))
+        or "**From:**" in raw[:500]
+        or bool(re.match(r"(From|Return-Path|Received|Delivered-To):", raw))
+    )
+
+
 def _discover_files(source: Path) -> list[Path]:
     if source.is_file():
         return [source]
@@ -481,8 +499,11 @@ def _preprocess_file(
     if max_file_bytes and size > max_file_bytes:
         return {"path": str(path), "status": "skipped", "reason": f"file_too_large:{size}", "bytes": size, "seconds": time.time() - started}
 
-    raw = path.read_text(errors="ignore")
-    text, email_meta = _extract_email_text(raw, path)
+    raw = path.read_text(encoding="utf-8", errors="ignore")
+    if _looks_like_email(path, raw):
+        text, email_meta = _extract_email_text(raw, path)
+    else:
+        text, email_meta = raw, {"source": str(path)}
     tokenizer = Tokenizer.load(tokenizer_name)
 
     # Structure-aware pre-chunking by file type.  The tokenizer remains the

tests/test_ingest_unicode.py

@@ -0,0 +1,45 @@
+from contextfit.cli import _looks_like_email, _preprocess_file
+from contextfit.core.tokenizer import Tokenizer
+
+SAMPLE_MD = (
+    "# Notizen Köln\n"
+    "\n"
+    "Grüße aus der Straße: ökonomisch, äußerst übermäßig — ß bleibt ß.\n"
+    "Beyond Latin-1: 日本語のテキスト und emoji ✅.\n"
+)
+
+
+def test_plain_markdown_is_not_treated_as_email(tmp_path):
+    path = tmp_path / "note.md"
+    path.write_text(SAMPLE_MD, encoding="utf-8")
+    assert not _looks_like_email(path, path.read_text(encoding="utf-8"))
+
+
+def test_eml_and_markdown_email_are_treated_as_email(tmp_path):
+    assert _looks_like_email(tmp_path / "mail.eml", "From: a@b.c\n\nbody\n")
+    assert _looks_like_email(
+        tmp_path / "mail.md", "# Email: Update\n**From:** a@b.c\n\n## Content\nhi\n"
+    )
+    assert _looks_like_email(
+        tmp_path / "raw.txt", "From: Alice <a@b.c>\nSubject: hi\n\nbody\n"
+    )
+
+
+def test_markdown_ingest_preserves_non_ascii(tmp_path):
+    path = tmp_path / "note.md"
+    path.write_text(SAMPLE_MD, encoding="utf-8")
+
+    result = _preprocess_file(
+        path,
+        tokenizer_name="cl100k_base",
+        chunk_size=64,
+        overlap=0,
+        max_file_bytes=0,
+        max_file_tokens=0,
+    )
+
+    assert result["status"] == "ok"
+    tokenizer = Tokenizer.load("cl100k_base")
+    text = "\n".join(tokenizer.decode(item["tokens"]) for item in result["token_items"])
+    for needle in ("Köln", "Grüße", "Straße", "äußerst", "日本語", "✅"):
+        assert needle in text