Add deterministic scoring baseline

.github/workflows/ci.yml

@@ -13,7 +13,8 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - name: Placeholder
+      - name: Test
         run: |
-          python --version
-          echo "Add linting and tests with the first implementation PR."
+          python -m pip install --upgrade pip
+          python -m pip install -e '.[dev]'
+          pytest

.gitignore

@@ -0,0 +1,6 @@
+.venv*/
+__pycache__/
+.pytest_cache/
+*.egg-info/
+dist/
+build/

README.md

@@ -26,3 +26,17 @@ This engine should never receive raw, non-anonymized submissions. Tests and exam
 ## First Milestone
 
 Implement a deterministic baseline scorer with fixtures, schema validation, and a report comparing output against hand-labeled examples.
+
+## Current Baseline
+
+This repo includes a deterministic keyword baseline for synthetic, anonymized records.
+It is intentionally transparent and marked as unvalidated for research claims.
+
+Run it locally:
+
+```bash
+python -m scoring_engine.cli examples/anonymized_conversation.json
+```
+
+The baseline outputs dimension scores, confidence values, evidence keywords, and a
+rubric version. It must only receive anonymized records.

examples/anonymized_conversation.json

@@ -0,0 +1,19 @@
+{
+  "schema_version": "0.1.0",
+  "conversation_id": "00000000-0000-4000-8000-000000000001",
+  "messages": [
+    {
+      "role": "user",
+      "content": "I feel anxious about my homework and I do not know how to tell my parent."
+    },
+    {
+      "role": "assistant",
+      "content": "Let's break the study plan into smaller steps and think about what support would feel safe."
+    }
+  ],
+  "redaction_report": {
+    "total": 0,
+    "by_type": {},
+    "redactions": []
+  }
+}

pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "makeaivisible-scoring-engine"
+version = "0.1.0"
+description = "Deterministic baseline scorer for anonymized Make AI Visible records."
+requires-python = ">=3.12"
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0"]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]

scoring_engine/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["score_record"]
+
+from .baseline import score_record

scoring_engine/baseline.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+RUBRIC_VERSION = "baseline-0.1.0"
+
+
+@dataclass(frozen=True)
+class DimensionRule:
+    name: str
+    keywords: tuple[str, ...]
+    description: str
+
+
+DIMENSIONS = (
+    DimensionRule(
+        "academic_support",
+        ("homework", "study", "essay", "exam", "class", "teacher"),
+        "Conversation includes schoolwork or learning support.",
+    ),
+    DimensionRule(
+        "emotional_support",
+        ("anxious", "sad", "stress", "worried", "lonely", "feel"),
+        "Conversation includes emotional support or wellbeing language.",
+    ),
+    DimensionRule(
+        "identity_or_relationships",
+        ("friend", "parent", "family", "crush", "relationship", "identity"),
+        "Conversation includes identity, family, or relationship topics.",
+    ),
+    DimensionRule(
+        "safety_sensitive",
+        ("harm", "unsafe", "secret", "threat", "hurt", "danger"),
+        "Conversation includes safety-sensitive terms that require review.",
+    ),
+)
+
+
+def _messages(record: dict[str, Any]) -> list[dict[str, str]]:
+    messages = record.get("messages")
+    if not isinstance(messages, list) or not messages:
+        raise ValueError("record must include a non-empty messages list")
+    return messages
+
+
+def score_record(record: dict[str, Any]) -> dict[str, Any]:
+    messages = _messages(record)
+    joined = "\n".join(str(message.get("content", "")) for message in messages).lower()
+
+    dimensions = []
+    for rule in DIMENSIONS:
+        matched = sorted({keyword for keyword in rule.keywords if keyword in joined})
+        score = min(1.0, len(matched) / 2)
+        dimensions.append(
+            {
+                "name": rule.name,
+                "score": score,
+                "confidence": 0.5 if matched else 0.2,
+                "evidence": matched,
+                "description": rule.description,
+            }
+        )
+
+    return {
+        "schema_version": "0.1.0",
+        "source_conversation_id": record.get("conversation_id"),
+        "rubric_version": RUBRIC_VERSION,
+        "dimensions": dimensions,
+        "validated_for_claims": False,
+    }

scoring_engine/cli.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from .baseline import score_record
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Score an anonymized conversation record.")
+    parser.add_argument("input", type=Path)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    result = score_record(json.loads(args.input.read_text()))
+    payload = json.dumps(result, indent=2) + "\n"
+    if args.output:
+        args.output.write_text(payload)
+    else:
+        print(payload, end="")
+
+
+if __name__ == "__main__":
+    main()

tests/test_baseline.py

@@ -0,0 +1,26 @@
+import json
+from pathlib import Path
+
+from scoring_engine import score_record
+
+
+def test_scores_anonymized_fixture() -> None:
+    fixture = json.loads(Path("examples/anonymized_conversation.json").read_text())
+
+    result = score_record(fixture)
+
+    assert result["source_conversation_id"] == fixture["conversation_id"]
+    assert result["rubric_version"] == "baseline-0.1.0"
+    assert result["validated_for_claims"] is False
+    scores = {item["name"]: item["score"] for item in result["dimensions"]}
+    assert scores["academic_support"] > 0
+    assert scores["emotional_support"] > 0
+
+
+def test_rejects_missing_messages() -> None:
+    try:
+        score_record({"conversation_id": "demo"})
+    except ValueError as error:
+        assert "messages" in str(error)
+    else:
+        raise AssertionError("expected ValueError")