diff --git a/bead/resources/adapters/unimorph.py b/bead/resources/adapters/unimorph.py index 3c7c91b..f65b920 100644 --- a/bead/resources/adapters/unimorph.py +++ b/bead/resources/adapters/unimorph.py @@ -52,6 +52,24 @@ def __init__(self, cache: AdapterCache | None = None) -> None: self.cache = cache self._datasets: dict[str, pd.DataFrame] = {} # Cache datasets by language + def _load_dataset(self, lang_code: str) -> pd.DataFrame: + """Load the raw ``(lemma, form, features)`` DataFrame for a language. + + Override this in a subclass to read a non-standard UniMorph file layout + (e.g. a language whose data ships extra columns). + + Parameters + ---------- + lang_code : str + ISO 639-3 language code. + + Returns + ------- + pd.DataFrame + DataFrame with ``lemma``, ``form``, and ``features`` columns. + """ + return load_dataset(lang_code) + def fetch_items( self, query: str | None = None, @@ -113,7 +131,7 @@ def fetch_items( try: # Load dataset for language (cached at instance level) if lang_code not in self._datasets: - self._datasets[lang_code] = load_dataset(lang_code) + self._datasets[lang_code] = self._load_dataset(lang_code) dataset = self._datasets[lang_code] diff --git a/gallery/ukr/argument_structure/Makefile b/gallery/ukr/argument_structure/Makefile new file mode 100644 index 0000000..a6826cd --- /dev/null +++ b/gallery/ukr/argument_structure/Makefile @@ -0,0 +1,207 @@ +# ============================================================================ +# Argument Structure Pipeline - Makefile (Ukrainian) +# ============================================================================ +# +# Quick start: +# make help - List targets +# make data - Run the experiment +# make data-quick - Smoke test over a few verbs +# make deployment - Build the jsPsych/JATOS experiments +# +# ============================================================================ + +PROJECT_ROOT := $(shell cd ../../.. && pwd) +PYTHON := uv run --project $(PROJECT_ROOT) python +# build_frequencies.py is the only script needing wordfreq, supplied per call +# so the pipeline itself carries no such dependency. +PYTHON_FREQ := uv run --project $(PROJECT_ROOT) --with wordfreq python +JSPSYCH_DIR := $(PROJECT_ROOT)/bead/deployment/jspsych + +LEXICONS_DIR := lexicons +TEMPLATES_DIR := templates +ITEMS_DIR := items +LISTS_DIR := lists +CACHE_DIR := .cache + +# How many verbs the experiment tests, chosen by frequency since VESUM is +# ordered alphabetically. At 100 each verb still contributes its six case +# contrasts across the configured lists. +VERBS := 100 +QUICK_VERBS := 20 + +BLUE := \033[0;34m +GREEN := \033[0;32m +YELLOW := \033[0;33m +RED := \033[0;31m +NC := \033[0m + +# ============================================================================ +# Main targets +# ============================================================================ + +.PHONY: help +help: ## List available targets + @echo "$(BLUE)Argument Structure Pipeline (Ukrainian)$(NC)" + @echo "" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " $(YELLOW)%-22s$(NC) %s\n", $$1, $$2}' + @echo "" + @echo "$(BLUE)Examples:$(NC)" + @echo " make data # the experiment ($(VERBS) verbs)" + @echo " make data-quick # smoke test ($(QUICK_VERBS) verbs)" + @echo " make deployment # every configured list" + @echo " make deployment-quick # 2 lists, for local testing" + @echo " make frequencies # regenerate the committed frequency table" + @echo "" + @echo "$(RED)clean-cache and clean-all discard the cached model scores,$(NC)" + @echo "$(RED)so the next 2afc-pairs run rescores every sentence.$(NC)" + @echo "" + +.PHONY: all +all: data deployment ## Run the pipeline end to end + @echo "$(GREEN)Pipeline complete$(NC)" + +.PHONY: data +data: lexicons templates fill-templates 2afc-pairs lists ## Generate all data for the experiment + @echo "$(GREEN)Data generated$(NC)" + +.PHONY: data-quick +data-quick: lexicons templates fill-templates-quick 2afc-pairs lists ## Generate all data over a few verbs + @echo "$(GREEN)Data generated$(NC)" + +# ============================================================================ +# Pipeline stages +# ============================================================================ + +.PHONY: lexicons +lexicons: ## [1/5] Build the verb and bleached noun lexicons from VESUM + @echo "$(BLUE)[1/5] Lexicons$(NC)" + $(PYTHON) generate_lexicons.py + +.PHONY: frequencies +frequencies: ## Regenerate resources/verb_frequencies.csv (needs wordfreq, run rarely) + @echo "$(BLUE)Verb frequencies$(NC)" + $(PYTHON_FREQ) build_frequencies.py + +.PHONY: templates +templates: ## [2/5] Build the five sentence frames + @echo "$(BLUE)[2/5] Templates$(NC)" + $(PYTHON) generate_templates.py + +.PHONY: fill-templates +fill-templates: ## [3/5] Fill the frames over the experiment verbs + @echo "$(BLUE)[3/5] Filling frames ($(VERBS) verbs)$(NC)" + $(PYTHON) fill_templates.py --limit $(VERBS) --by-frequency + +.PHONY: fill-templates-quick +fill-templates-quick: ## [3/5] Fill the frames over a few verbs + @echo "$(BLUE)[3/5] Filling frames ($(QUICK_VERBS) verbs)$(NC)" + $(PYTHON) fill_templates.py --limit $(QUICK_VERBS) --by-frequency + +.PHONY: 2afc-pairs +2afc-pairs: ## [4/5] Score the sentences and build the minimal pairs + @echo "$(BLUE)[4/5] 2AFC pairs$(NC)" + @echo "$(YELLOW)Scoring loads a 12B model and is slow on first run; scores are cached$(NC)" + $(PYTHON) create_2afc_pairs.py + +.PHONY: lists +lists: ## [5/5] Partition the pairs into experiment lists + @echo "$(BLUE)[5/5] Experiment lists$(NC)" + $(PYTHON) generate_lists.py + +# ============================================================================ +# Deployment +# ============================================================================ + +.PHONY: jspsych-build +jspsych-build: ## Compile the jsPsych assets the generator needs + @echo "$(BLUE)Building jsPsych assets$(NC)" + cd $(JSPSYCH_DIR) && npm install --silent && npm run build && npm run build:gallery + +.PHONY: deployment +deployment: jspsych-build ## Build every configured list and package for JATOS + @echo "$(BLUE)Deployment (all configured lists)$(NC)" + $(PYTHON) generate_deployment.py + +.PHONY: deployment-quick +deployment-quick: jspsych-build ## Build 2 experiments for local testing + @echo "$(BLUE)Deployment (2 lists)$(NC)" + $(PYTHON) generate_deployment.py --n-lists 2 + @echo " Open deployment/local/list_01/index.html" + +.PHONY: deployment-local +deployment-local: jspsych-build ## Build the local version only, skipping JATOS packaging + @echo "$(BLUE)Deployment (local only)$(NC)" + $(PYTHON) generate_deployment.py --no-jatos + +# ============================================================================ +# Quality +# ============================================================================ + +.PHONY: lint +lint: ## Lint the pipeline scripts + @echo "$(BLUE)Linting$(NC)" + $(PYTHON) -m ruff check \ + build_frequencies.py \ + generate_lexicons.py \ + generate_templates.py \ + fill_templates.py \ + create_2afc_pairs.py \ + generate_lists.py \ + generate_deployment.py \ + utils/ + +.PHONY: check +check: lint ## Run all quality checks + +# ============================================================================ +# Inspection +# ============================================================================ + +.PHONY: show-stats +show-stats: ## Show counts for the generated files + @echo "$(BLUE)Generated data$(NC)" + @for f in $(LEXICONS_DIR)/*.jsonl $(TEMPLATES_DIR)/*.jsonl \ + $(ITEMS_DIR)/*.jsonl $(LISTS_DIR)/*.jsonl; do \ + if [ -f "$$f" ]; then \ + printf " %-34s %8s lines %6s\n" "$$f" \ + "$$(wc -l < $$f | tr -d ' ')" \ + "$$(ls -lh $$f | awk '{print $$5}')"; \ + fi \ + done + @if [ -d deployment/jatos ]; then \ + printf " %-34s %8s archives\n" "deployment/jatos" \ + "$$(ls deployment/jatos/*.jzip 2>/dev/null | wc -l | tr -d ' ')"; \ + fi + +.PHONY: show-config +show-config: ## Show the key configuration values + @$(PYTHON) -c "import yaml; c = yaml.safe_load(open('config.yaml')); \ + m = [x for x in c['items']['models'] if x.get('use_for_scoring')][0]; \ + print('language: ', c['project']['language_code']); \ + print('scoring model: ', m['name'], '(' + m['type'] + ', ' + m['device'] + ')'); \ + print('frames: ', 5); \ + print('lists: ', c['lists']['n_lists'], 'x', c['lists']['items_per_list']); \ + print('deploying: ', c['deployment']['n_lists_to_deploy'], 'lists')" + +# ============================================================================ +# Cleaning +# ============================================================================ + +.PHONY: clean +clean: ## Remove generated items, lists and deployment + @rm -f $(ITEMS_DIR)/*.jsonl $(LISTS_DIR)/*.jsonl + @rm -rf deployment + @echo "$(GREEN)Cleaned items, lists and deployment$(NC)" + +.PHONY: clean-cache +clean-cache: ## Discard the cached scores, forcing a full rescore next run + @rm -rf $(CACHE_DIR) + @echo "$(RED)Cached scores discarded; the next run rescores every sentence$(NC)" + +.PHONY: clean-all +clean-all: clean clean-cache ## Remove every generated file AND the cached scores + @rm -f $(LEXICONS_DIR)/*.jsonl $(TEMPLATES_DIR)/*.jsonl + @echo "$(RED)All generated data removed (resources/ kept)$(NC)" + +.DEFAULT_GOAL := help diff --git a/gallery/ukr/argument_structure/README.md b/gallery/ukr/argument_structure/README.md new file mode 100644 index 0000000..7ebe655 --- /dev/null +++ b/gallery/ukr/argument_structure/README.md @@ -0,0 +1,152 @@ +# Ukrainian Argument Structure + +## Quick Start + +```bash +make data-quick # Smoke test over 20 verbs +make data # The experiment (100 verbs) +make deployment # Build the jsPsych experiments and package them for JATOS +make help # Show all available targets +``` + + +## Pipeline Stages + +| Step | Script | Make target | +|------|--------|-------------| +| 1. Lexicons | `generate_lexicons.py` | `make lexicons` | +| 2. Templates | `generate_templates.py` | `make templates` | +| 3. Fill templates | `fill_templates.py` | `make fill-templates` / `make fill-templates-quick` | +| 4. 2AFC pairs | `create_2afc_pairs.py` | `make 2afc-pairs` | +| 5. Lists | `generate_lists.py` | `make lists` | +| 6. Deployment | `generate_deployment.py` | `make deployment` / `make deployment-quick` | + +--- + +### Stage 1: Lexicons (`make lexicons`) + +Writes two files under `lexicons/`. + +`verbs.jsonl` holds present-tense verb forms. `bleached_nouns.jsonl` holds one +semantically light noun per class, in each of the cases the frames need: + +| Lemma | Class | Role | Gloss | +|-------|-------|------|-------| +| людина | animate | subject, object | person | +| гурт | group | object | band, group | +| інструмент | inanimate_object | object | instrument | +| зразок | abstract | object | sample | +| двір | location | object | yard | +| день | temporal | object | day | +| шматок | quantity | object | piece | +| випадок | event | object | incident | + + +Ukrainian case syncretism means a single form often realizes several cells. When +a case has more than one candidate form, the generator prefers a form that +realizes only its own cell. Two collisions survive that filter and are recorded +on the item as `case_collides_with`, so downstream stages can see them: `людини` +also realizes `ACC.PL`, and `випадку` also realizes `DAT.SG`. + +--- + +### Stage 2: Templates (`make templates`) + +Writes `templates/generic_frames.jsonl` with five frames: + +| Frame | Slots | +|-------|-------| +| `intransitive` | `subj_nom`, `verb` | +| `obj_acc` | `subj_nom`, `obj_acc`, `verb` | +| `obj_gen` | `subj_nom`, `obj_gen`, `verb` | +| `obj_dat` | `subj_nom`, `obj_dat`, `verb` | +| `obj_ins` | `subj_nom`, `obj_ins`, `verb` | + + +The verb slot carries a stoplist read from `template.verb_stoplist` in +`config.yaml`. + +--- + +### Stage 3: Fill templates (`make fill-templates`) + +Fills every slot exhaustively and writes `items/filled.jsonl`. Filling is +dispatched through bead's `MixedFillingStrategy`, so an individual slot can be +switched to masked-LM selection in `config.yaml` without touching the script. +With the current bleached noun set every slot is exhaustive, and no model is +loaded. + +`--limit N` caps the number of distinct verb lemmas. Pass `--by-frequency` +alongside it, as the Makefile does, to keep the *most frequent* lemmas. VESUM is +ordered alphabetically, so slicing off the head would otherwise yield rare verbs +rather than everyday ones. + +--- + +### Stage 4: 2AFC pairs (`make 2afc-pairs`) + +Scores every object-bearing sentence with a language model, then builds two +kinds of pair and writes them to `items/2afc_pairs.jsonl`. + +**Case contrasts** group sentences that share a verb and an object noun, and +pair them across cases. Because the noun and verb are held fixed, the two +sentences differ only in the object's case form. Pairs whose two sentences +render identically are dropped, which is how the remaining syncretism from stage +1 is handled. + +**Anchor contrasts** pair a test verb against a verb whose government is already +known, in the same frame and with the same noun: + +| Case | Anchor verb | Gloss | +|------|-------------|-------| +| ACC | вивчати | study | +| GEN | стосуватися | concern | +| DAT | радіти | rejoice | +| INS | цікавитися | to be interested in | + +Anchors put every verb on a scale shared across verbs, and they expose verbs +that fit no frame at all, which a purely within-verb contrast cannot detect. +The four were chosen to combine sensibly with all of the bleached nouns. + +Both pair types record the score difference between their two sentences, and are +then assigned difficulty quantiles stratified by contrast type, so that easy and +hard pairs are distinguishable when lists are built. + +`config.yaml` offers a masked and a causal model; masked models are scored +by pseudo-log-likelihood and causal ones by sentence log probability. + +--- + +### Stages 5-6: Lists and Deployment + +`make lists` partitions pairs into lists; +`make deployment-quick` builds two lists for local testing; +`make deployment-local` skips JATOS packaging entirely. + +--- + +## Configuration + +Everything lives in `config.yaml`. + +`resources/verb_frequencies.csv` is a committed resource, derived from +`wordfreq`, that fixes the verb ranking across runs. Regenerate it with +`make frequencies`, which supplies `wordfreq` for that one call so the pipeline +itself does not depend on it. Frequencies are taken from the infinitive rather +than summed over the paradigm, because summing lets a paradigm absorb unrelated +homographs (колоти would absorb коли, and маяти would absorb має). + +## Utility Targets + +```bash +make show-stats # Counts and sizes for the generated files +make show-config # Key configuration values +make lint # Ruff over the pipeline scripts +make clean # Remove generated items, lists, and deployment +make clean-cache # Discard the cached model scores +make clean-all # Remove every generated file and the cached scores +``` + +`clean-cache` and `clean-all` throw away the cached sentence scores, which means +the next `make 2afc-pairs` rescores every sentence from scratch. + diff --git a/gallery/ukr/argument_structure/build_frequencies.py b/gallery/ukr/argument_structure/build_frequencies.py new file mode 100644 index 0000000..a069650 --- /dev/null +++ b/gallery/ukr/argument_structure/build_frequencies.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Write the verb frequency table used to choose which verbs to test. + +Run once, with wordfreq supplied for the invocation:: + + uv run --project --with wordfreq python build_frequencies.py + +Frequency is the Zipf value of the infinitive, from the wordfreq package +(Ukrainian, which wordfreq keys by the ISO 639-1 code "uk"). Summing over a +verb's whole paradigm was tried and rejected: infinitives in -ти are distinctive, +while finite forms collide with frequent homographs, so колоти collects the count +of the conjunction коли and маяти collects має from мати. + +The generated CSV is committed so the pipeline needs no wordfreq dependency and +the ranking stays fixed across runs. Unattested lemmas are omitted and read +as 0.0. +""" + +from __future__ import annotations + +import csv +from pathlib import Path + +from wordfreq import zipf_frequency + +from bead.cli.display import print_header, print_success +from bead.resources.lexicon import Lexicon + +BASE_DIR = Path(__file__).parent +WORDFREQ_LANGUAGE = "uk" + + +def main() -> None: + """Rank every verb lemma by the Zipf frequency of its infinitive.""" + print_header("Verb Frequencies (wordfreq)") + + lexicon = Lexicon.from_jsonl(str(BASE_DIR / "lexicons" / "verbs.jsonl"), "verbs") + lemmas = {item.lemma for item in lexicon.items} + attested = sorted( + ((lemma, zipf_frequency(lemma, WORDFREQ_LANGUAGE)) for lemma in lemmas), + key=lambda row: (-row[1], row[0]), + ) + attested = [(lemma, zipf) for lemma, zipf in attested if zipf > 0.0] + + output_path = BASE_DIR / "resources" / "verb_frequencies.csv" + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["lemma", "zipf"]) + writer.writerows((lemma, f"{zipf:.2f}") for lemma, zipf in attested) + + print_success( + f"Wrote {len(attested):,} attested lemmas of {len(lemmas):,} to {output_path}" + ) + + +if __name__ == "__main__": + main() diff --git a/gallery/ukr/argument_structure/config.yaml b/gallery/ukr/argument_structure/config.yaml new file mode 100644 index 0000000..8ca9259 --- /dev/null +++ b/gallery/ukr/argument_structure/config.yaml @@ -0,0 +1,194 @@ +# ============================================================================ +# Ukrainian argument structure pipeline configuration +# ============================================================================ +# Design principle: test every verb in every frame (full cross-product), so +# both licensed and unlicensed verb-frame combinations are judged. +# ============================================================================ + +project: + name: "argument_structure" + language_code: "ukr" + +paths: + cache_dir: ".cache" + 2afc_pairs: "items/2afc_pairs.jsonl" + experiment_lists: "lists/experiment_lists.jsonl" + +resources: + templates: + - path: "templates/generic_frames.jsonl" + name: "generic_frames" + description: "Argument-structure frames constructed manually" + + lexicons: + - path: "lexicons/verbs.jsonl" + name: "verbs" + description: "Present-tense verbs from VESUM" + + - path: "lexicons/bleached_nouns.jsonl" + name: "bleached_nouns" + description: "Bleached subject and object nouns in their case forms" + +template: + # Strategy type: "exhaustive", "mlm", or "mixed" + filling_strategy: "mixed" + + # Output path for filled templates + output_path: "items/filled.jsonl" + + # Excluded from the verb slot. + verb_stoplist: + - "бути" # copula + - "могти" # modal + - "смеркати" # impersonal + + # Masked language model settings, used only by slots whose strategy is "mlm". + # Not yet finalized; unused while every slot below is exhaustive. + mlm: + model_name: "youscan/ukr-roberta-base" + beam_size: 5 + top_k: 10 + device: "cpu" + cache_enabled: true + + # Per-slot strategies: + # - exhaustive: use all lexicon items that satisfy the slot's constraints + # - mlm: let the model select contextually appropriate fillers + # max_fills: keep the top-N items by model probability + # enforce_unique: each beam hypothesis uses different items for this slot + slot_strategies: + subj_nom: + strategy: "exhaustive" + description: "The bleached subject in the nominative" + + verb: + strategy: "exhaustive" + description: "All present-tense verbs (cross-product with all frames)" + + obj_acc: + strategy: "exhaustive" + description: "Accusative objects" + + obj_gen: + strategy: "exhaustive" + description: "Genitive objects" + + obj_dat: + strategy: "exhaustive" + description: "Dative objects" + + obj_ins: + strategy: "exhaustive" + description: "Instrumental objects" + +items: + judgment_type: "forced_choice" + n_alternatives: 2 + + # Sentence-acceptability scoring models. Exactly one entry sets + # use_for_scoring: true (override with --model). Masked models are scored by + # pseudo-log-likelihood, causal models by sentence log-probability. + # Accuracy at picking the correct case for the anchor verbs: lapa 92%, roberta 62%. + models: + - name: "lapa-llm/lapa-12b-pt" + type: "causal" + device: "mps" + dtype: "bfloat16" + use_for_scoring: true + + - name: "benjamin/roberta-large-wechsel-ukrainian" + type: "masked" + device: "cpu" + use_for_scoring: false + + # Minimal pairs contrast the case of a verb's object while holding the verb + # and the noun fixed. Pairs whose two sentences render identically (case + # syncretism) are dropped. + construction: + group_by: ["verb_lemma", "object_lemma"] + drop_identical: true + + # Verbs with known government, paired against a test verb in the same frame + # so verbs share a scale. Chosen to combine with every bleached noun. + anchors: + ACC: "вивчати" + GEN: "стосуватися" + DAT: "радіти" + INS: "цікавитися" + +lists: + strategy: "balanced" + n_lists: 16 + items_per_list: 50 + quantile_bins: 10 + random_seed: 42 + + # "item" is the pair's metadata dict, so properties are read as item['key']. + # The balance target_counts also fix how many of each pair type are drawn. + constraints: + - type: "balance" + property_expression: "item['pair_type']" + target_counts: + case_contrast: 40 + anchor_contrast: 10 + + - type: "uniqueness" + property_expression: "item['verb']" + + batch_constraints: + - type: "coverage" + property_expression: "item['contrast']" + target_values: + - "ACC-GEN" + - "ACC-DAT" + - "ACC-INS" + - "DAT-GEN" + - "GEN-INS" + - "DAT-INS" + - "anchor-ACC" + - "anchor-GEN" + - "anchor-DAT" + - "anchor-INS" + min_coverage: 1.0 + + - type: "min_occurrence" + property_expression: "item['quantile']" + min_occurrences: 20 + + - type: "diversity" + property_expression: "item['verb']" + max_lists_per_value: 6 + +deployment: + platform: "jatos" + n_lists_to_deploy: 16 + random_seed: 42 + output_dir: "deployment" + + experiment: + title: "Оцінювання прийнятності речень" + description: "Порівняйте два речення й оберіть те, яке звучить природніше" + instructions: >- + У кожному завданні ви побачите два речення. + Оберіть те, яке звучить для вас більш природно. + prompt: "Яке речення звучить природніше?" + estimated_duration_minutes: 15 + + jspsych: + version: "8.0.0" + use_jatos: true + prolific_completion_code: null + randomize_order: true + randomize_choices: true + + participants: + n_per_list: 30 + qualifications: + - "Native Ukrainian speaker" + - "Age 18+" + - "Approval rate > 95%" + payment_usd: 2.50 + +logging: + level: "WARNING" + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" diff --git a/gallery/ukr/argument_structure/create_2afc_pairs.py b/gallery/ukr/argument_structure/create_2afc_pairs.py new file mode 100644 index 0000000..c7bd8d4 --- /dev/null +++ b/gallery/ukr/argument_structure/create_2afc_pairs.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +"""Create 2AFC minimal pairs from the filled sentences. + +Loads the filled sentences, scores each with the selected language model, then +pairs sentences that share a verb and object noun but differ in the object's +case, so each pair isolates the case the verb governs. Pairs are tagged with the +score difference and assigned difficulty quantiles, and written to the configured +output path. +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path +from uuid import UUID + +import yaml +from utils.scoring import MaskedLanguageModelScorer + +from bead.cli.display import ( + create_live_status, + display_file_stats, + print_header, + print_info, + print_success, + print_warning, +) +from bead.items.forced_choice import ( + create_filtered_forced_choice_items, + create_forced_choice_item, +) +from bead.items.item import Item +from bead.items.scoring import ItemScorer, LanguageModelScorer +from bead.lists.stratification import assign_quantiles_by_uuid +from bead.templates.filler import FilledTemplate + +BASE_DIR = Path(__file__).parent + + +def load_config(path: Path) -> dict: + """Load the YAML configuration file. + + Parameters + ---------- + path : Path + Path to the configuration file. + + Returns + ------- + dict + Parsed configuration. + """ + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) + + +def load_filled(path: Path, limit: int | None = None) -> list[FilledTemplate]: + """Load filled sentences from JSONL. + + Parameters + ---------- + path : Path + Path to the filled-sentences file. + limit : int | None + Read at most this many lines. + + Returns + ------- + list[FilledTemplate] + The loaded filled sentences. + """ + filled: list[FilledTemplate] = [] + with path.open(encoding="utf-8") as f: + for i, line in enumerate(f): + if limit is not None and i >= limit: + break + filled.append(FilledTemplate.model_validate_json(line)) + return filled + + +def to_items(filled: list[FilledTemplate]) -> list[Item]: + """Convert filled sentences with an object into scorable items. + + Intransitive sentences (no object) are dropped, since the case contrast is + defined only over object-bearing frames. + + Parameters + ---------- + filled : list[FilledTemplate] + The filled sentences. + + Returns + ------- + list[Item] + One item per object-bearing sentence, tagged with verb, object class, + and case. + """ + items: list[Item] = [] + for ft in filled: + verb = ft.slot_fillers.get("verb") + obj = next( + (f for name, f in ft.slot_fillers.items() if name.startswith("obj_")), + None, + ) + if verb is None or obj is None: + continue + items.append( + Item( + item_template_id=UUID(ft.template_id), + rendered_elements={"text": ft.rendered_text}, + item_metadata={ + "filled_template_id": str(ft.id), + "template_name": ft.template_name, + "verb_lemma": verb.lemma, + "object_lemma": obj.lemma, + "object_class": obj.features.get("semantic_class"), + "case": obj.features.get("case"), + }, + ) + ) + return items + + +def select_model(config: dict, override: str | None) -> dict: + """Return the model config to score with. + + Parameters + ---------- + config : dict + Parsed configuration. + override : str | None + Model name to force, ignoring the ``use_for_scoring`` flag. + + Returns + ------- + dict + The selected model's configuration entry. + + Raises + ------ + ValueError + If ``override`` names no configured model. + """ + models = config["items"]["models"] + if override is not None: + for model in models: + if model["name"] == override: + return model + raise ValueError(f"No configured model named '{override}'") + for model in models: + if model.get("use_for_scoring"): + return model + return models[0] + + +def build_scorer(model: dict, cache_dir: Path) -> ItemScorer: + """Build the scorer matching a model's type. + + Parameters + ---------- + model : dict + A model configuration entry (``name``, ``type``, ``device``). + cache_dir : Path + Directory for the model output cache. + + Returns + ------- + ItemScorer + A masked or causal scorer. + """ + device = model.get("device", "cpu") + if model["type"] == "masked": + return MaskedLanguageModelScorer( + model_name=model["name"], cache_dir=cache_dir, device=device + ) + return LanguageModelScorer( + model_name=model["name"], + cache_dir=cache_dir, + device=device, + dtype=model.get("dtype", "auto"), + ) + + +def make_pairs(items: list[Item]) -> list[Item]: + """Pair object-bearing items by verb and object class, contrasting case. + + Groups items sharing a verb and object noun, forms every within-group pair, + and drops pairs whose two sentences render identically (case syncretism). + + Parameters + ---------- + items : list[Item] + Scored, object-bearing items. + + Returns + ------- + list[Item] + Forced-choice pairs tagged with case contrast and score difference. + """ + by_id = {str(item.id): item for item in items} + + def text_of(item: Item) -> str: + """Return an item's rendered sentence text.""" + return item.rendered_elements.get("text", "") + + pairs = create_filtered_forced_choice_items( + items=items, + group_by=lambda item: ( + item.item_metadata["verb_lemma"], + item.item_metadata["object_lemma"], + ), + n_alternatives=2, + combination_filter=lambda combo: text_of(combo[0]) != text_of(combo[1]), + extract_text=text_of, + ) + + enriched: list[Item] = [] + for pair in pairs: + ids = pair.item_metadata.get("source_item_ids", ()) + first, second = by_id.get(ids[0]), by_id.get(ids[1]) + if first is None or second is None: + continue + case_a = first.item_metadata["case"] + case_b = second.item_metadata["case"] + score_a = first.item_metadata["lm_score"] + score_b = second.item_metadata["lm_score"] + pair = pair.with_( + item_metadata={ + **pair.item_metadata, + "pair_type": "case_contrast", + "verb": first.item_metadata["verb_lemma"], + "object_lemma": first.item_metadata["object_lemma"], + "object_class": first.item_metadata["object_class"], + "case_a": case_a, + "case_b": case_b, + "contrast": "-".join(sorted([str(case_a), str(case_b)])), + "lm_score_a": score_a, + "lm_score_b": score_b, + "lm_score_diff": abs(score_a - score_b), + } + ) + enriched.append(pair) + return enriched + + +def make_anchor_pairs(items: list[Item], anchors: dict[str, str]) -> list[Item]: + """Pair each verb against the anchor verb for its case, noun held fixed. + + The anchor's government is known, so the comparison puts test verbs on a + scale shared across verbs and exposes verbs that fit no frame at all. + + Parameters + ---------- + items : list[Item] + Scored, object-bearing items. + anchors : dict[str, str] + Anchor lemma for each case. + + Returns + ------- + list[Item] + Forced-choice pairs contrasting a verb with its case's anchor. + """ + by_key = { + ( + item.item_metadata["verb_lemma"], + item.item_metadata["case"], + item.item_metadata["object_lemma"], + ): item + for item in items + } + + pairs: list[Item] = [] + for item in items: + case_name = item.item_metadata["case"] + verb = item.item_metadata["verb_lemma"] + anchor_lemma = anchors.get(str(case_name)) + if anchor_lemma is None or verb == anchor_lemma: + continue + key = (anchor_lemma, case_name, item.item_metadata["object_lemma"]) + anchor = by_key.get(key) + if anchor is None: + continue + + text = item.rendered_elements.get("text", "") + anchor_text = anchor.rendered_elements.get("text", "") + if text == anchor_text: + continue + score = item.item_metadata["lm_score"] + anchor_score = anchor.item_metadata["lm_score"] + pairs.append( + create_forced_choice_item( + text, + anchor_text, + metadata={ + "pair_type": "anchor_contrast", + "verb": verb, + "anchor": anchor_lemma, + "case": case_name, + "contrast": f"anchor-{case_name}", + "object_lemma": item.item_metadata["object_lemma"], + "object_class": item.item_metadata["object_class"], + "lm_score_a": score, + "lm_score_b": anchor_score, + "lm_score_diff": abs(score - anchor_score), + "source_item_ids": (str(item.id), str(anchor.id)), + }, + ) + ) + return pairs + + +def main( + config_path: Path, + limit: int | None = None, + output: Path | None = None, + model_override: str | None = None, +) -> None: + """Score sentences, build case-contrast pairs, and write them out. + + Parameters + ---------- + config_path : Path + Path to the configuration file. + limit : int | None + Read at most this many filled sentences. + output : Path | None + Override the output path from the config. + model_override : str | None + Score with this model name instead of the configured default. + """ + config = load_config(config_path) + logging.basicConfig( + level=getattr(logging, config["logging"]["level"]), + format=config["logging"]["format"], + ) + + print_header("2AFC Pair Creation") + + cache_dir = BASE_DIR / config["paths"]["cache_dir"] + filled_path = BASE_DIR / config["template"]["output_path"] + output_path = output or (BASE_DIR / config["paths"]["2afc_pairs"]) + + filled = load_filled(filled_path, limit) + items = to_items(filled) + print_success(f"Loaded {len(items):,} object-bearing sentences") + if not items: + print_warning("No object-bearing sentences to pair.") + return + + model = select_model(config, model_override) + scorer = build_scorer(model, cache_dir) + print_info(f"Scoring with {model['name']} ({model['type']})") + with create_live_status("Scoring sentences..."): + scores = scorer.score_batch(items) + items = [ + item.with_(item_metadata={**item.item_metadata, "lm_score": score}) + for item, score in zip(items, scores, strict=True) + ] + + case_pairs = make_pairs(items) + anchor_pairs = make_anchor_pairs(items, config["items"]["construction"]["anchors"]) + pairs = case_pairs + anchor_pairs + print_success( + f"Built {len(case_pairs):,} case-contrast and " + f"{len(anchor_pairs):,} anchor pairs" + ) + if not pairs: + print_warning("No pairs were created.") + return + + n_quantiles = config["lists"]["quantile_bins"] + quantiles = assign_quantiles_by_uuid( + item_ids=[pair.id for pair in pairs], + item_metadata={pair.id: pair.item_metadata for pair in pairs}, + property_key="lm_score_diff", + n_quantiles=n_quantiles, + stratify_by_key="contrast", + ) + pairs = [ + pair.with_(item_metadata={**pair.item_metadata, "quantile": quantiles[pair.id]}) + for pair in pairs + ] + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as f: + for pair in pairs: + f.write(pair.model_dump_json() + "\n") + display_file_stats(output_path, len(pairs), "pairs") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Create 2AFC case-contrast pairs from the filled sentences." + ) + parser.add_argument( + "--config", + type=Path, + default=BASE_DIR / "config.yaml", + help="Path to the configuration file.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Read at most this many filled sentences (for testing).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Override the output path from the config.", + ) + parser.add_argument( + "--model", + type=str, + default=None, + help="Score with this model name instead of the configured default.", + ) + args = parser.parse_args() + main( + config_path=args.config, + limit=args.limit, + output=args.output, + model_override=args.model, + ) diff --git a/gallery/ukr/argument_structure/fill_templates.py b/gallery/ukr/argument_structure/fill_templates.py new file mode 100644 index 0000000..b31cef7 --- /dev/null +++ b/gallery/ukr/argument_structure/fill_templates.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Fill the Ukrainian frames with lexical items. + +Loads the frames and lexicons, then for each frame produces every combination of +items satisfying the slot constraints, rendered into a sentence, and writes them +to the configured output path. Per-slot strategies come from ``config.yaml``; +every slot is exhaustive by default, but a slot may instead be filled by a +masked language model. +""" + +from __future__ import annotations + +import argparse +import logging +from collections.abc import Iterable +from itertools import islice +from pathlib import Path + +import yaml +from utils.frequency import most_frequent +from utils.renderers import UkrainianRenderer + +from bead.cli.display import ( + create_progress, + display_file_stats, + print_header, + print_info, + print_success, + print_warning, +) +from bead.data.serialization import write_jsonlines +from bead.resources.lexicon import Lexicon +from bead.resources.template_collection import TemplateCollection +from bead.templates.filler import FilledTemplate +from bead.templates.resolver import ConstraintResolver +from bead.templates.strategies import MixedFillingStrategy, StrategyConfig + +BASE_DIR = Path(__file__).parent +LANGUAGE = "ukr" + + +def load_config(path: Path) -> dict: + """Load the YAML configuration file. + + Parameters + ---------- + path : Path + Path to the configuration file. + + Returns + ------- + dict + Parsed configuration. + """ + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) + + +def build_strategy(config: dict) -> MixedFillingStrategy: + """Build the filling strategy from the config's per-slot strategies. + + The masked language model is loaded only when some slot requests it, so an + all-exhaustive run needs no model. + + Parameters + ---------- + config : dict + Parsed configuration. + + Returns + ------- + MixedFillingStrategy + Strategy dispatching each slot to its configured filler. + """ + slot_configs = config["template"]["slot_strategies"] + mlm = config["template"]["mlm"] + resolver = ConstraintResolver() + + model_adapter = None + cache = None + if any(sc["strategy"] == "mlm" for sc in slot_configs.values()): + from bead.templates.adapters.cache import ModelOutputCache # noqa: PLC0415 + from bead.templates.adapters.huggingface import ( # noqa: PLC0415 + HuggingFaceMLMAdapter, + ) + + print_info(f"Loading MLM model: {mlm['model_name']}...") + model_adapter = HuggingFaceMLMAdapter( + model_name=mlm["model_name"], device=mlm.get("device", "cpu") + ) + model_adapter.load_model() + cache = ModelOutputCache(cache_dir=BASE_DIR / config["paths"]["cache_dir"]) + + slot_strategies: dict[str, tuple[str, StrategyConfig]] = {} + for slot_name, slot_config in slot_configs.items(): + if slot_config["strategy"] == "mlm": + settings: StrategyConfig = { + "resolver": resolver, + "model_adapter": model_adapter, + "cache": cache, + "beam_size": mlm.get("beam_size", 5), + "top_k": mlm.get("top_k", 10), + } + if "max_fills" in slot_config: + settings["max_fills"] = slot_config["max_fills"] + if "enforce_unique" in slot_config: + settings["enforce_unique"] = slot_config["enforce_unique"] + slot_strategies[slot_name] = ("mlm", settings) + else: + slot_strategies[slot_name] = (slot_config["strategy"], {}) + + return MixedFillingStrategy(slot_strategies=slot_strategies) + + +def limit_verbs( + lexicon: Lexicon, + limit: int, + *, + by_frequency: bool = False, + keep: Iterable[str] = (), +) -> Lexicon: + """Return a lexicon keeping only ``limit`` unique verb lemmas. + + Parameters + ---------- + lexicon : Lexicon + Verb lexicon. + limit : int + Number of unique lemmas to keep. + by_frequency : bool + Keep the most frequent lemmas instead of the first ones, since VESUM is + alphabetical and the first lemmas are rare rather than everyday verbs. + keep : Iterable[str] + Lemmas retained whatever the limit, for the anchor verbs the pair stage + compares against. + + Returns + ------- + Lexicon + Lexicon restricted to ``limit`` lemmas plus ``keep``. + """ + if by_frequency: + allowed = set(most_frequent((item.lemma for item in lexicon.items), limit)) + else: + lemmas: list[str] = [] + for item in lexicon.items: + if item.lemma not in lemmas: + lemmas.append(item.lemma) + if len(lemmas) >= limit: + break + allowed = set(lemmas) + allowed |= set(keep) + return lexicon.filter(lambda item: item.lemma in allowed) + + +def main( + config_path: Path, + limit: int | None = None, + max_per_template: int | None = None, + output: Path | None = None, + *, + by_frequency: bool = False, +) -> None: + """Fill every frame and write the sentences to the output path. + + Parameters + ---------- + config_path : Path + Path to the configuration file. + limit : int | None + Keep only this many unique verb lemmas (for quick runs). + max_per_template : int | None + Keep at most this many sentences per frame. + output : Path | None + Override the output path from the config. + by_frequency : bool + Select the most frequent verbs rather than the first ones. + """ + config = load_config(config_path) + logging.basicConfig( + level=getattr(logging, config["logging"]["level"]), + format=config["logging"]["format"], + ) + + print_header("Template Filling") + + templates_config = config["resources"]["templates"][0] + templates = list( + TemplateCollection.from_jsonl( + str(BASE_DIR / templates_config["path"]), templates_config["name"] + ) + ) + print_success(f"Loaded {len(templates)} frames") + + lexicons: list[Lexicon] = [] + for lex_config in config["resources"]["lexicons"]: + lexicon = Lexicon.from_jsonl( + str(BASE_DIR / lex_config["path"]), lex_config["name"] + ) + if limit is not None and lex_config["name"] == "verbs": + anchors = config["items"]["construction"]["anchors"].values() + lexicon = limit_verbs( + lexicon, limit, by_frequency=by_frequency, keep=anchors + ) + selection = "most frequent" if by_frequency else "first" + print_warning( + f"Limited verbs to the {selection} {limit} lemmas " + f"plus {len(set(anchors))} anchors ({len(lexicon)} forms)" + ) + lexicons.append(lexicon) + print_info(f"Loaded {len(lexicon)} items from {lex_config['name']}") + + strategy = build_strategy(config) + renderer = UkrainianRenderer() + + filled: list[FilledTemplate] = [] + counts: dict[str, int] = {} + with create_progress() as progress: + task = progress.add_task("Filling frames", total=len(templates)) + for template in templates: + combos = strategy.generate_from_template( + template=template, lexicons=lexicons, language_code=LANGUAGE + ) + if max_per_template is not None: + combos = islice(combos, max_per_template) + + start = len(filled) + for combo in combos: + rendered = renderer.render( + template.template_string, combo, template.slots + ) + filled.append( + FilledTemplate( + template_id=str(template.id), + template_name=template.name, + slot_fillers=combo, + rendered_text=rendered, + strategy_name="mixed", + template_slots={ + name: slot.required for name, slot in template.slots.items() + }, + ) + ) + counts[template.name] = len(filled) - start + progress.advance(task) + + for name, count in counts.items(): + print_info(f"{name}: {count:,} sentences") + print_success(f"Filled {len(filled):,} sentences") + + output_path = output or (BASE_DIR / config["template"]["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + write_jsonlines(filled, output_path) + display_file_stats(output_path, len(filled), "sentences") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Fill the Ukrainian frames with lexical items." + ) + parser.add_argument( + "--config", + type=Path, + default=BASE_DIR / "config.yaml", + help="Path to the configuration file.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Keep only this many unique verb lemmas (for testing).", + ) + parser.add_argument( + "--max-per-template", + type=int, + default=None, + help="Keep at most this many sentences per frame.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Override the output path from the config.", + ) + parser.add_argument( + "--by-frequency", + action="store_true", + help="Select the most frequent verbs rather than the first ones.", + ) + args = parser.parse_args() + main( + config_path=args.config, + limit=args.limit, + max_per_template=args.max_per_template, + output=args.output, + by_frequency=args.by_frequency, + ) diff --git a/gallery/ukr/argument_structure/generate_deployment.py b/gallery/ukr/argument_structure/generate_deployment.py new file mode 100644 index 0000000..4e821df --- /dev/null +++ b/gallery/ukr/argument_structure/generate_deployment.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Generate the jsPsych/JATOS deployment from the experiment lists. + +Builds one jsPsych experiment per selected list in two versions: a standalone +local build for testing in a browser, and a JATOS build packaged as a ``.jzip`` +archive for upload. Text shown to participants comes from the config. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from random import Random +from uuid import UUID + +import yaml + +from bead.cli.display import ( + create_progress, + print_header, + print_info, + print_success, +) +from bead.deployment.distribution import ( + DistributionStrategyType, + ListDistributionStrategy, +) +from bead.deployment.jatos.exporter import JATOSExporter +from bead.deployment.jspsych.config import ( + ChoiceConfig, + ExperimentConfig, + InstructionsConfig, +) +from bead.deployment.jspsych.generator import JsPsychExperimentGenerator +from bead.items.item import Item +from bead.items.item_template import ItemTemplate, PresentationSpec, TaskSpec +from bead.lists.experiment_list import ExperimentList + +BASE_DIR = Path(__file__).parent +INSTRUCTIONS_PLUGIN = ( + '' +) + + +def load_config(path: Path) -> dict: + """Load the YAML configuration file. + + Parameters + ---------- + path : Path + Path to the configuration file. + + Returns + ------- + dict + Parsed configuration. + """ + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) + + +def load_lists(path: Path) -> list[ExperimentList]: + """Load the experiment lists from JSONL. + + Parameters + ---------- + path : Path + Path to the lists file. + + Returns + ------- + list[ExperimentList] + The loaded lists. + """ + with path.open(encoding="utf-8") as f: + return [ExperimentList.model_validate_json(line) for line in f if line.strip()] + + +def load_pairs(path: Path) -> dict[UUID, Item]: + """Load the 2AFC pairs, keyed by the id the lists reference. + + Parameters + ---------- + path : Path + Path to the pairs file. + + Returns + ------- + dict[UUID, Item] + Pairs indexed by their id. + """ + pairs: dict[UUID, Item] = {} + with path.open(encoding="utf-8") as f: + for line in f: + if line.strip(): + item = Item.model_validate_json(line) + pairs[item.id] = item + return pairs + + +def build_item_template(prompt: str) -> ItemTemplate: + """Build the template the generator renders each pair through. + + Parameters + ---------- + prompt : str + Question shown above the two options. + + Returns + ------- + ItemTemplate + Template for the forced-choice items. + """ + return ItemTemplate( + name="2afc_case_contrast", + description="Two-alternative forced choice over object case", + judgment_type="acceptability", + task_type="forced_choice", + task_spec=TaskSpec(prompt=prompt, options=["Речення 1", "Речення 2"]), + presentation_spec=PresentationSpec(mode="static"), + ) + + +def _inject_instructions_plugin(html_path: Path) -> None: + """Add the jsPsych instructions plugin tag when the generator omits it. + + Parameters + ---------- + html_path : Path + Generated ``index.html`` to patch. + """ + html = html_path.read_text(encoding="utf-8") + if "plugin-instructions" in html: + return + preload = '