From cce2b96ce1b86c8435bb60e128f35eb3e6a2968e Mon Sep 17 00:00:00 2001 From: Alp Date: Sun, 12 Jul 2026 11:41:41 -0700 Subject: [PATCH] Document PII detector gap tests --- docs/pii-detector-playbook.md | 18 ++++++++++++++++++ tests/test_redaction.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/docs/pii-detector-playbook.md b/docs/pii-detector-playbook.md index 15eb415..4121dce 100644 --- a/docs/pii-detector-playbook.md +++ b/docs/pii-detector-playbook.md @@ -31,3 +31,21 @@ These are visible in `DETECTOR_GAPS` so contributors can pick them up intentiona Each gap needs examples, false-positive checks, and privacy review before it should be enabled for real data. + +`tests/test_redaction.py` includes expected-failure tests for these gaps. They should +only be converted into passing tests when the detector behavior, limitations, and review +requirements are documented. + +## Challenge #4 Acceptance Progress + +| Identifier type | Status | +| --- | --- | +| Emails | Active deterministic detector and tests | +| Phone numbers | Active deterministic detector and tests | +| Usernames | Active deterministic detector and tests | +| URLs | Active deterministic detector and tests | +| Account IDs | Active deterministic detector and tests | +| Names | Expected-failure synthetic gap test | +| Schools | Expected-failure synthetic gap test | +| Addresses | Expected-failure synthetic gap test | +| Indirect identifiers | Expected-failure synthetic gap test | diff --git a/tests/test_redaction.py b/tests/test_redaction.py index 1820590..5748247 100644 --- a/tests/test_redaction.py +++ b/tests/test_redaction.py @@ -48,3 +48,39 @@ def test_common_near_misses_are_not_redacted(source: str) -> None: assert result == source assert redactions == [] + + +@pytest.mark.xfail(reason="Challenge #4 gap: person-name detection needs expert review") +def test_future_person_name_detection_gap() -> None: + result, redactions = redact_text("My name is Maya Chen.", 0, defaultdict(int)) + + assert "Maya Chen" not in result + assert redactions[0].entity_type == "PERSON_NAME" + + +@pytest.mark.xfail(reason="Challenge #4 gap: school detection needs curated examples") +def test_future_school_detection_gap() -> None: + result, redactions = redact_text("I go to Cedar Ridge Secondary.", 0, defaultdict(int)) + + assert "Cedar Ridge Secondary" not in result + assert redactions[0].entity_type == "SCHOOL" + + +@pytest.mark.xfail(reason="Challenge #4 gap: address detection needs locale-aware rules") +def test_future_address_detection_gap() -> None: + result, redactions = redact_text("Please pick me up at 42 Maple Street.", 0, defaultdict(int)) + + assert "42 Maple Street" not in result + assert redactions[0].entity_type == "ADDRESS" + + +@pytest.mark.xfail(reason="Challenge #4 gap: indirect identifiers need policy decisions") +def test_future_indirect_identifier_detection_gap() -> None: + result, redactions = redact_text( + "I am the only grade 10 student on the robotics team in my town.", + 0, + defaultdict(int), + ) + + assert "only grade 10 student" not in result + assert redactions[0].entity_type == "INDIRECT_IDENTIFIER"