From dd0f7008a7b1d8aa801b0ced90820a86d30d1b21 Mon Sep 17 00:00:00 2001 From: Alp Date: Sun, 12 Jul 2026 11:41:39 -0700 Subject: [PATCH] Add deterministic scoring baseline --- .github/workflows/ci.yml | 7 +-- .gitignore | 6 +++ README.md | 14 ++++++ examples/anonymized_conversation.json | 19 +++++++ pyproject.toml | 11 ++++ scoring_engine/__init__.py | 3 ++ scoring_engine/baseline.py | 72 +++++++++++++++++++++++++++ scoring_engine/cli.py | 25 ++++++++++ tests/test_baseline.py | 26 ++++++++++ 9 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 .gitignore create mode 100644 examples/anonymized_conversation.json create mode 100644 pyproject.toml create mode 100644 scoring_engine/__init__.py create mode 100644 scoring_engine/baseline.py create mode 100644 scoring_engine/cli.py create mode 100644 tests/test_baseline.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6c8b11..132f187 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Placeholder + - name: Test run: | - python --version - echo "Add linting and tests with the first implementation PR." + python -m pip install --upgrade pip + python -m pip install -e '.[dev]' + pytest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0fd23f --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.venv*/ +__pycache__/ +.pytest_cache/ +*.egg-info/ +dist/ +build/ diff --git a/README.md b/README.md index 206f84a..ab15e85 100644 --- a/README.md +++ b/README.md @@ -26,3 +26,17 @@ This engine should never receive raw, non-anonymized submissions. Tests and exam ## First Milestone Implement a deterministic baseline scorer with fixtures, schema validation, and a report comparing output against hand-labeled examples. + +## Current Baseline + +This repo includes a deterministic keyword baseline for synthetic, anonymized records. +It is intentionally transparent and marked as unvalidated for research claims. + +Run it locally: + +```bash +python -m scoring_engine.cli examples/anonymized_conversation.json +``` + +The baseline outputs dimension scores, confidence values, evidence keywords, and a +rubric version. It must only receive anonymized records. diff --git a/examples/anonymized_conversation.json b/examples/anonymized_conversation.json new file mode 100644 index 0000000..356a715 --- /dev/null +++ b/examples/anonymized_conversation.json @@ -0,0 +1,19 @@ +{ + "schema_version": "0.1.0", + "conversation_id": "00000000-0000-4000-8000-000000000001", + "messages": [ + { + "role": "user", + "content": "I feel anxious about my homework and I do not know how to tell my parent." + }, + { + "role": "assistant", + "content": "Let's break the study plan into smaller steps and think about what support would feel safe." + } + ], + "redaction_report": { + "total": 0, + "by_type": {}, + "redactions": [] + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2173b1e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "makeaivisible-scoring-engine" +version = "0.1.0" +description = "Deterministic baseline scorer for anonymized Make AI Visible records." +requires-python = ">=3.12" + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[tool.pytest.ini_options] +pythonpath = ["."] diff --git a/scoring_engine/__init__.py b/scoring_engine/__init__.py new file mode 100644 index 0000000..743ce8c --- /dev/null +++ b/scoring_engine/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["score_record"] + +from .baseline import score_record diff --git a/scoring_engine/baseline.py b/scoring_engine/baseline.py new file mode 100644 index 0000000..0d9c5b5 --- /dev/null +++ b/scoring_engine/baseline.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +RUBRIC_VERSION = "baseline-0.1.0" + + +@dataclass(frozen=True) +class DimensionRule: + name: str + keywords: tuple[str, ...] + description: str + + +DIMENSIONS = ( + DimensionRule( + "academic_support", + ("homework", "study", "essay", "exam", "class", "teacher"), + "Conversation includes schoolwork or learning support.", + ), + DimensionRule( + "emotional_support", + ("anxious", "sad", "stress", "worried", "lonely", "feel"), + "Conversation includes emotional support or wellbeing language.", + ), + DimensionRule( + "identity_or_relationships", + ("friend", "parent", "family", "crush", "relationship", "identity"), + "Conversation includes identity, family, or relationship topics.", + ), + DimensionRule( + "safety_sensitive", + ("harm", "unsafe", "secret", "threat", "hurt", "danger"), + "Conversation includes safety-sensitive terms that require review.", + ), +) + + +def _messages(record: dict[str, Any]) -> list[dict[str, str]]: + messages = record.get("messages") + if not isinstance(messages, list) or not messages: + raise ValueError("record must include a non-empty messages list") + return messages + + +def score_record(record: dict[str, Any]) -> dict[str, Any]: + messages = _messages(record) + joined = "\n".join(str(message.get("content", "")) for message in messages).lower() + + dimensions = [] + for rule in DIMENSIONS: + matched = sorted({keyword for keyword in rule.keywords if keyword in joined}) + score = min(1.0, len(matched) / 2) + dimensions.append( + { + "name": rule.name, + "score": score, + "confidence": 0.5 if matched else 0.2, + "evidence": matched, + "description": rule.description, + } + ) + + return { + "schema_version": "0.1.0", + "source_conversation_id": record.get("conversation_id"), + "rubric_version": RUBRIC_VERSION, + "dimensions": dimensions, + "validated_for_claims": False, + } diff --git a/scoring_engine/cli.py b/scoring_engine/cli.py new file mode 100644 index 0000000..f22ca07 --- /dev/null +++ b/scoring_engine/cli.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .baseline import score_record + + +def main() -> None: + parser = argparse.ArgumentParser(description="Score an anonymized conversation record.") + parser.add_argument("input", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + result = score_record(json.loads(args.input.read_text())) + payload = json.dumps(result, indent=2) + "\n" + if args.output: + args.output.write_text(payload) + else: + print(payload, end="") + + +if __name__ == "__main__": + main() diff --git a/tests/test_baseline.py b/tests/test_baseline.py new file mode 100644 index 0000000..42c9f2c --- /dev/null +++ b/tests/test_baseline.py @@ -0,0 +1,26 @@ +import json +from pathlib import Path + +from scoring_engine import score_record + + +def test_scores_anonymized_fixture() -> None: + fixture = json.loads(Path("examples/anonymized_conversation.json").read_text()) + + result = score_record(fixture) + + assert result["source_conversation_id"] == fixture["conversation_id"] + assert result["rubric_version"] == "baseline-0.1.0" + assert result["validated_for_claims"] is False + scores = {item["name"]: item["score"] for item in result["dimensions"]} + assert scores["academic_support"] > 0 + assert scores["emotional_support"] > 0 + + +def test_rejects_missing_messages() -> None: + try: + score_record({"conversation_id": "demo"}) + except ValueError as error: + assert "messages" in str(error) + else: + raise AssertionError("expected ValueError")