Adds a Ukrainian argument structure gallery example

bead/resources/adapters/unimorph.py

@@ -52,6 +52,24 @@ def __init__(self, cache: AdapterCache | None = None) -> None:
         self.cache = cache
         self._datasets: dict[str, pd.DataFrame] = {}  # Cache datasets by language
 
+    def _load_dataset(self, lang_code: str) -> pd.DataFrame:
+        """Load the raw ``(lemma, form, features)`` DataFrame for a language.
+
+        Override this in a subclass to read a non-standard UniMorph file layout
+        (e.g. a language whose data ships extra columns).
+
+        Parameters
+        ----------
+        lang_code : str
+            ISO 639-3 language code.
+
+        Returns
+        -------
+        pd.DataFrame
+            DataFrame with ``lemma``, ``form``, and ``features`` columns.
+        """
+        return load_dataset(lang_code)
+
     def fetch_items(
         self,
         query: str | None = None,
@@ -113,7 +131,7 @@ def fetch_items(
         try:
             # Load dataset for language (cached at instance level)
             if lang_code not in self._datasets:
-                self._datasets[lang_code] = load_dataset(lang_code)
+                self._datasets[lang_code] = self._load_dataset(lang_code)
 
             dataset = self._datasets[lang_code]
 

gallery/ukr/argument_structure/Makefile

@@ -0,0 +1,207 @@
+# ============================================================================
+# Argument Structure Pipeline - Makefile (Ukrainian)
+# ============================================================================
+#
+# Quick start:
+#   make help          - List targets
+#   make data          - Run the experiment
+#   make data-quick    - Smoke test over a few verbs
+#   make deployment    - Build the jsPsych/JATOS experiments
+#
+# ============================================================================
+
+PROJECT_ROOT := $(shell cd ../../.. && pwd)
+PYTHON       := uv run --project $(PROJECT_ROOT) python
+# build_frequencies.py is the only script needing wordfreq, supplied per call
+# so the pipeline itself carries no such dependency.
+PYTHON_FREQ  := uv run --project $(PROJECT_ROOT) --with wordfreq python
+JSPSYCH_DIR  := $(PROJECT_ROOT)/bead/deployment/jspsych
+
+LEXICONS_DIR := lexicons
+TEMPLATES_DIR := templates
+ITEMS_DIR    := items
+LISTS_DIR    := lists
+CACHE_DIR    := .cache
+
+# How many verbs the experiment tests, chosen by frequency since VESUM is
+# ordered alphabetically. At 100 each verb still contributes its six case
+# contrasts across the configured lists.
+VERBS        := 100
+QUICK_VERBS  := 20
+
+BLUE   := \033[0;34m
+GREEN  := \033[0;32m
+YELLOW := \033[0;33m
+RED    := \033[0;31m
+NC     := \033[0m
+
+# ============================================================================
+# Main targets
+# ============================================================================
+
+.PHONY: help
+help: ## List available targets
+	@echo "$(BLUE)Argument Structure Pipeline (Ukrainian)$(NC)"
+	@echo ""
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+		awk 'BEGIN {FS = ":.*?## "}; {printf "  $(YELLOW)%-22s$(NC) %s\n", $$1, $$2}'
+	@echo ""
+	@echo "$(BLUE)Examples:$(NC)"
+	@echo "  make data                 # the experiment ($(VERBS) verbs)"
+	@echo "  make data-quick           # smoke test ($(QUICK_VERBS) verbs)"
+	@echo "  make deployment           # every configured list"
+	@echo "  make deployment-quick     # 2 lists, for local testing"
+	@echo "  make frequencies          # regenerate the committed frequency table"
+	@echo ""
+	@echo "$(RED)clean-cache and clean-all discard the cached model scores,$(NC)"
+	@echo "$(RED)so the next 2afc-pairs run rescores every sentence.$(NC)"
+	@echo ""
+
+.PHONY: all
+all: data deployment ## Run the pipeline end to end
+	@echo "$(GREEN)Pipeline complete$(NC)"
+
+.PHONY: data
+data: lexicons templates fill-templates 2afc-pairs lists ## Generate all data for the experiment
+	@echo "$(GREEN)Data generated$(NC)"
+
+.PHONY: data-quick
+data-quick: lexicons templates fill-templates-quick 2afc-pairs lists ## Generate all data over a few verbs
+	@echo "$(GREEN)Data generated$(NC)"
+
+# ============================================================================
+# Pipeline stages
+# ============================================================================
+
+.PHONY: lexicons
+lexicons: ## [1/5] Build the verb and bleached noun lexicons from VESUM
+	@echo "$(BLUE)[1/5] Lexicons$(NC)"
+	$(PYTHON) generate_lexicons.py
+
+.PHONY: frequencies
+frequencies: ## Regenerate resources/verb_frequencies.csv (needs wordfreq, run rarely)
+	@echo "$(BLUE)Verb frequencies$(NC)"
+	$(PYTHON_FREQ) build_frequencies.py
+
+.PHONY: templates
+templates: ## [2/5] Build the five sentence frames
+	@echo "$(BLUE)[2/5] Templates$(NC)"
+	$(PYTHON) generate_templates.py
+
+.PHONY: fill-templates
+fill-templates: ## [3/5] Fill the frames over the experiment verbs
+	@echo "$(BLUE)[3/5] Filling frames ($(VERBS) verbs)$(NC)"
+	$(PYTHON) fill_templates.py --limit $(VERBS) --by-frequency
+
+.PHONY: fill-templates-quick
+fill-templates-quick: ## [3/5] Fill the frames over a few verbs
+	@echo "$(BLUE)[3/5] Filling frames ($(QUICK_VERBS) verbs)$(NC)"
+	$(PYTHON) fill_templates.py --limit $(QUICK_VERBS) --by-frequency
+
+.PHONY: 2afc-pairs
+2afc-pairs: ## [4/5] Score the sentences and build the minimal pairs
+	@echo "$(BLUE)[4/5] 2AFC pairs$(NC)"
+	@echo "$(YELLOW)Scoring loads a 12B model and is slow on first run; scores are cached$(NC)"
+	$(PYTHON) create_2afc_pairs.py
+
+.PHONY: lists
+lists: ## [5/5] Partition the pairs into experiment lists
+	@echo "$(BLUE)[5/5] Experiment lists$(NC)"
+	$(PYTHON) generate_lists.py
+
+# ============================================================================
+# Deployment
+# ============================================================================
+
+.PHONY: jspsych-build
+jspsych-build: ## Compile the jsPsych assets the generator needs
+	@echo "$(BLUE)Building jsPsych assets$(NC)"
+	cd $(JSPSYCH_DIR) && npm install --silent && npm run build && npm run build:gallery
+
+.PHONY: deployment
+deployment: jspsych-build ## Build every configured list and package for JATOS
+	@echo "$(BLUE)Deployment (all configured lists)$(NC)"
+	$(PYTHON) generate_deployment.py
+
+.PHONY: deployment-quick
+deployment-quick: jspsych-build ## Build 2 experiments for local testing
+	@echo "$(BLUE)Deployment (2 lists)$(NC)"
+	$(PYTHON) generate_deployment.py --n-lists 2
+	@echo "  Open deployment/local/list_01/index.html"
+
+.PHONY: deployment-local
+deployment-local: jspsych-build ## Build the local version only, skipping JATOS packaging
+	@echo "$(BLUE)Deployment (local only)$(NC)"
+	$(PYTHON) generate_deployment.py --no-jatos
+
+# ============================================================================
+# Quality
+# ============================================================================
+
+.PHONY: lint
+lint: ## Lint the pipeline scripts
+	@echo "$(BLUE)Linting$(NC)"
+	$(PYTHON) -m ruff check \
+		build_frequencies.py \
+		generate_lexicons.py \
+		generate_templates.py \
+		fill_templates.py \
+		create_2afc_pairs.py \
+		generate_lists.py \
+		generate_deployment.py \
+		utils/
+
+.PHONY: check
+check: lint ## Run all quality checks
+
+# ============================================================================
+# Inspection
+# ============================================================================
+
+.PHONY: show-stats
+show-stats: ## Show counts for the generated files
+	@echo "$(BLUE)Generated data$(NC)"
+	@for f in $(LEXICONS_DIR)/*.jsonl $(TEMPLATES_DIR)/*.jsonl \
+	          $(ITEMS_DIR)/*.jsonl $(LISTS_DIR)/*.jsonl; do \
+		if [ -f "$$f" ]; then \
+			printf "  %-34s %8s lines  %6s\n" "$$f" \
+				"$$(wc -l < $$f | tr -d ' ')" \
+				"$$(ls -lh $$f | awk '{print $$5}')"; \
+		fi \
+	done
+	@if [ -d deployment/jatos ]; then \
+		printf "  %-34s %8s archives\n" "deployment/jatos" \
+			"$$(ls deployment/jatos/*.jzip 2>/dev/null | wc -l | tr -d ' ')"; \
+	fi
+
+.PHONY: show-config
+show-config: ## Show the key configuration values
+	@$(PYTHON) -c "import yaml; c = yaml.safe_load(open('config.yaml')); \
+		m = [x for x in c['items']['models'] if x.get('use_for_scoring')][0]; \
+		print('language:      ', c['project']['language_code']); \
+		print('scoring model: ', m['name'], '(' + m['type'] + ', ' + m['device'] + ')'); \
+		print('frames:        ', 5); \
+		print('lists:         ', c['lists']['n_lists'], 'x', c['lists']['items_per_list']); \
+		print('deploying:     ', c['deployment']['n_lists_to_deploy'], 'lists')"
+
+# ============================================================================
+# Cleaning
+# ============================================================================
+
+.PHONY: clean
+clean: ## Remove generated items, lists and deployment
+	@rm -f $(ITEMS_DIR)/*.jsonl $(LISTS_DIR)/*.jsonl
+	@rm -rf deployment
+	@echo "$(GREEN)Cleaned items, lists and deployment$(NC)"
+
+.PHONY: clean-cache
+clean-cache: ## Discard the cached scores, forcing a full rescore next run
+	@rm -rf $(CACHE_DIR)
+	@echo "$(RED)Cached scores discarded; the next run rescores every sentence$(NC)"
+
+.PHONY: clean-all
+clean-all: clean clean-cache ## Remove every generated file AND the cached scores
+	@rm -f $(LEXICONS_DIR)/*.jsonl $(TEMPLATES_DIR)/*.jsonl
+	@echo "$(RED)All generated data removed (resources/ kept)$(NC)"
+
+.DEFAULT_GOAL := help

gallery/ukr/argument_structure/README.md

@@ -0,0 +1,152 @@
+# Ukrainian Argument Structure
+
+## Quick Start
+
+```bash
+make data-quick    # Smoke test over 20 verbs
+make data          # The experiment (100 verbs)
+make deployment    # Build the jsPsych experiments and package them for JATOS
+make help          # Show all available targets
+```
+
+
+## Pipeline Stages
+
+| Step | Script | Make target |
+|------|--------|-------------|
+| 1. Lexicons | `generate_lexicons.py` | `make lexicons` |
+| 2. Templates | `generate_templates.py` | `make templates` |
+| 3. Fill templates | `fill_templates.py` | `make fill-templates` / `make fill-templates-quick` |
+| 4. 2AFC pairs | `create_2afc_pairs.py` | `make 2afc-pairs` |
+| 5. Lists | `generate_lists.py` | `make lists` |
+| 6. Deployment | `generate_deployment.py` | `make deployment` / `make deployment-quick` |
+
+---
+
+### Stage 1: Lexicons (`make lexicons`)
+
+Writes two files under `lexicons/`.
+
+`verbs.jsonl` holds present-tense verb forms. `bleached_nouns.jsonl` holds one
+semantically light noun per class, in each of the cases the frames need:
+
+| Lemma | Class | Role | Gloss |
+|-------|-------|------|-------|
+| людина | animate | subject, object | person |
+| гурт | group | object | band, group |
+| інструмент | inanimate_object | object | instrument |
+| зразок | abstract | object | sample |
+| двір | location | object | yard |
+| день | temporal | object | day |
+| шматок | quantity | object | piece |
+| випадок | event | object | incident |
+
+
+Ukrainian case syncretism means a single form often realizes several cells. When
+a case has more than one candidate form, the generator prefers a form that
+realizes only its own cell. Two collisions survive that filter and are recorded
+on the item as `case_collides_with`, so downstream stages can see them: `людини`
+also realizes `ACC.PL`, and `випадку` also realizes `DAT.SG`.
+
+---
+
+### Stage 2: Templates (`make templates`)
+
+Writes `templates/generic_frames.jsonl` with five frames:
+
+| Frame | Slots |
+|-------|-------|
+| `intransitive` | `subj_nom`, `verb` |
+| `obj_acc` | `subj_nom`, `obj_acc`, `verb` |
+| `obj_gen` | `subj_nom`, `obj_gen`, `verb` |
+| `obj_dat` | `subj_nom`, `obj_dat`, `verb` |
+| `obj_ins` | `subj_nom`, `obj_ins`, `verb` |
+
+
+The verb slot carries a stoplist read from `template.verb_stoplist` in
+`config.yaml`.
+
+---
+
+### Stage 3: Fill templates (`make fill-templates`)
+
+Fills every slot exhaustively and writes `items/filled.jsonl`. Filling is
+dispatched through bead's `MixedFillingStrategy`, so an individual slot can be
+switched to masked-LM selection in `config.yaml` without touching the script.
+With the current bleached noun set every slot is exhaustive, and no model is
+loaded.
+
+`--limit N` caps the number of distinct verb lemmas. Pass `--by-frequency`
+alongside it, as the Makefile does, to keep the *most frequent* lemmas. VESUM is
+ordered alphabetically, so slicing off the head would otherwise yield rare verbs
+rather than everyday ones.
+
+---
+
+### Stage 4: 2AFC pairs (`make 2afc-pairs`)
+
+Scores every object-bearing sentence with a language model, then builds two
+kinds of pair and writes them to `items/2afc_pairs.jsonl`.
+
+**Case contrasts** group sentences that share a verb and an object noun, and
+pair them across cases. Because the noun and verb are held fixed, the two
+sentences differ only in the object's case form. Pairs whose two sentences
+render identically are dropped, which is how the remaining syncretism from stage
+1 is handled.
+
+**Anchor contrasts** pair a test verb against a verb whose government is already
+known, in the same frame and with the same noun:
+
+| Case | Anchor verb | Gloss |
+|------|-------------|-------|
+| ACC | вивчати | study |
+| GEN | стосуватися | concern |
+| DAT | радіти | rejoice |
+| INS | цікавитися | to be interested in |
+
+Anchors put every verb on a scale shared across verbs, and they expose verbs
+that fit no frame at all, which a purely within-verb contrast cannot detect.
+The four were chosen to combine sensibly with all of the bleached nouns.
+
+Both pair types record the score difference between their two sentences, and are
+then assigned difficulty quantiles stratified by contrast type, so that easy and
+hard pairs are distinguishable when lists are built.
+
+`config.yaml` offers a masked and a causal model; masked models are scored
+by pseudo-log-likelihood and causal ones by sentence log probability.
+
+---
+
+### Stages 5-6: Lists and Deployment
+
+`make lists` partitions pairs into lists;
+`make deployment-quick` builds two lists for local testing;
+`make deployment-local` skips JATOS packaging entirely.
+
+---
+
+## Configuration
+
+Everything lives in `config.yaml`.
+
+`resources/verb_frequencies.csv` is a committed resource, derived from
+`wordfreq`, that fixes the verb ranking across runs. Regenerate it with
+`make frequencies`, which supplies `wordfreq` for that one call so the pipeline
+itself does not depend on it. Frequencies are taken from the infinitive rather
+than summed over the paradigm, because summing lets a paradigm absorb unrelated
+homographs (колоти would absorb коли, and маяти would absorb має).
+
+## Utility Targets
+
+```bash
+make show-stats    # Counts and sizes for the generated files
+make show-config   # Key configuration values
+make lint          # Ruff over the pipeline scripts
+make clean         # Remove generated items, lists, and deployment
+make clean-cache   # Discard the cached model scores
+make clean-all     # Remove every generated file and the cached scores
+```
+
+`clean-cache` and `clean-all` throw away the cached sentence scores, which means
+the next `make 2afc-pairs` rescores every sentence from scratch.
+