From 34ac5050f6092aa82e1fac31e9b497941139e4fc Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 19:19:52 -0600 Subject: [PATCH 1/9] feat: add ordering-matters demo Three agent groups apply identical rules (sense, move, share) in different orders to demonstrate that execution order drives divergent emergent behavior. Includes NetLogo model with BehaviorSpace experiment, LLM rule-inference template, Python analysis script with 15 passing tests, and config/README. --- demos/ordering-matters/README.md | 76 ++ demos/ordering-matters/analysis.py | 188 +++++ demos/ordering-matters/config | 45 ++ demos/ordering-matters/ordering-matters.nlogo | 728 ++++++++++++++++++ .../rule-inference-template.yaml | 23 + demos/ordering-matters/tests/__init__.py | 0 demos/ordering-matters/tests/test_analysis.py | 167 ++++ 7 files changed, 1227 insertions(+) create mode 100644 demos/ordering-matters/README.md create mode 100644 demos/ordering-matters/analysis.py create mode 100644 demos/ordering-matters/config create mode 100644 demos/ordering-matters/ordering-matters.nlogo create mode 100644 demos/ordering-matters/rule-inference-template.yaml create mode 100644 demos/ordering-matters/tests/__init__.py create mode 100644 demos/ordering-matters/tests/test_analysis.py diff --git a/demos/ordering-matters/README.md b/demos/ordering-matters/README.md new file mode 100644 index 0000000..c23c485 --- /dev/null +++ b/demos/ordering-matters/README.md @@ -0,0 +1,76 @@ +# Ordering Matters Demo + +Demonstrates that applying identical rules in different orders produces measurably different emergent behavior in agent-based models. + +## Concept + +Three groups of foraging agents each have the same three behavioral rules: + +| Rule | Description | +|------|-------------| +| **SENSE** | Detect nearest food within sensor range | +| **MOVE** | Navigate toward food, follow advice, or random walk | +| **SHARE** | Communicate food locations with nearby agents | + +The groups differ **only** in execution order: + +| Group | Color | Order | Tendency | +|-------|-------|-------|----------| +| A | Red circles | sense → move → share | Reacts to own observations first | +| B | Blue squares | share → sense → move | Communicates before acting | +| C | Green triangles | move → share → sense | Acts impulsively, reflects later | + +## Files + +| File | Purpose | +|------|---------| +| `ordering-matters.nlogo` | NetLogo model with interface and BehaviorSpace experiment | +| `rule-inference-template.yaml` | LLM prompt template for inferring rule orderings from data | +| `config` | LLM provider configuration (OpenAI, Anthropic, Gemini, Ollama) | +| `analysis.py` | Python script to analyze exported simulation data | +| `tests/test_analysis.py` | Pytest suite for the analysis script | + +## Quick Start + +1. Edit `config` with your LLM provider credentials (or disable `use-llm?` in the model) +2. Open `ordering-matters.nlogo` in NetLogo 6.4+ +3. Click **setup** then **go-forever** +4. Watch the plot diverge as rule ordering drives different outcomes +5. Click **export-data** to save CSV, then analyze: + +```bash +python3 analysis.py ordering-matters-output.csv --plot results.png +``` + +## Running Without LLM + +The model works without any LLM provider. Turn off the `use-llm?` switch and agents will share raw coordinates instead of natural-language messages. This mode is useful for fast batch experiments via BehaviorSpace. + +## Analysis + +The `analysis.py` script reads exported CSV data and produces: + +- Per-group metrics (food collected, food rate, energy efficiency) +- Pairwise effect sizes between groups +- Summary report identifying the best-performing ordering +- Optional comparison plot (`--plot output.png`) + +### Running Tests + +```bash +cd demos/ordering-matters +python3 -m pytest tests/ -v +``` + +## Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `num-agents` | 18 | Total agents (divided into 3 equal groups) | +| `food-count` | 80 | Initial food patches | +| `sensor-range` | 4 | How far agents can detect food | +| `speed` | 1.0 | Distance agents move per tick | +| `comm-range` | 6 | Range for sharing information | +| `share-interval` | 5 | Ticks between communication attempts | +| `respawn-food?` | on | Whether food regenerates periodically | +| `respawn-interval` | 20 | Ticks between food respawn waves | diff --git a/demos/ordering-matters/analysis.py b/demos/ordering-matters/analysis.py new file mode 100644 index 0000000..9983aa1 --- /dev/null +++ b/demos/ordering-matters/analysis.py @@ -0,0 +1,188 @@ +# ABOUTME: Analyzes ordering-matters simulation CSV output to quantify how +# ABOUTME: rule execution order affects emergent agent behavior across groups. + +""" +Ordering-Matters Analysis +========================= +Parses NetLogo simulation exports and computes per-group metrics, +pairwise effect sizes, and summary reports. Optionally generates +comparison plots. + +Usage: + python3 analysis.py [--plot output.png] +""" + +import argparse +import sys +from pathlib import Path + +import pandas as pd +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +REQUIRED_COLUMNS = { + "tick", "group_a_food", "group_b_food", "group_c_food", + "group_a_energy", "group_b_energy", "group_c_energy", +} + +GROUP_LABELS = { + "a": "Group A (sense→move→share)", + "b": "Group B (share→sense→move)", + "c": "Group C (move→share→sense)", +} + + +def load_simulation_data(filepath: str) -> pd.DataFrame: + """Load and validate a simulation output CSV. + + Raises FileNotFoundError if path is invalid, ValueError if required + columns are absent. + """ + path = Path(filepath) + if not path.exists(): + raise FileNotFoundError(f"No such file: {filepath}") + + df = pd.read_csv(filepath) + missing = REQUIRED_COLUMNS - set(df.columns) + if missing: + raise ValueError(f"Missing required columns: {sorted(missing)}") + + for col in df.columns: + df[col] = pd.to_numeric(df[col]) + + return df + + +def compute_group_metrics(df: pd.DataFrame) -> dict: + """Compute per-group summary metrics from simulation data. + + Returns dict keyed by group letter ("a", "b", "c") with fields: + final_food, food_rate, final_energy, energy_spent, efficiency + """ + last = df.iloc[-1] + num_ticks = max(int(last["tick"]), 1) + initial_energy = float(df.iloc[0]["group_a_energy"]) # same start + + metrics = {} + for grp in ("a", "b", "c"): + food = float(last[f"group_{grp}_food"]) + energy = float(last[f"group_{grp}_energy"]) + spent = initial_energy - energy + metrics[grp] = { + "final_food": food, + "food_rate": food / num_ticks, + "final_energy": energy, + "energy_spent": spent, + "efficiency": food / spent if spent > 0 else 0.0, + } + return metrics + + +def compute_effect_size(metrics: dict) -> dict: + """Compute pairwise differences between groups. + + Returns dict with keys "a_vs_b", "a_vs_c", "b_vs_c", each containing + food_diff, efficiency_diff, and energy_diff. + """ + pairs = [("a", "b"), ("a", "c"), ("b", "c")] + effects = {} + for g1, g2 in pairs: + m1, m2 = metrics[g1], metrics[g2] + effects[f"{g1}_vs_{g2}"] = { + "food_diff": m1["final_food"] - m2["final_food"], + "efficiency_diff": m1["efficiency"] - m2["efficiency"], + "energy_diff": m1["energy_spent"] - m2["energy_spent"], + } + return effects + + +def generate_report(metrics: dict, effects: dict) -> str: + """Produce a human-readable summary report.""" + lines = [] + lines.append("=" * 60) + lines.append(" Ordering-Matters Simulation Analysis Report") + lines.append("=" * 60) + lines.append("") + + # Per-group results + for grp in ("a", "b", "c"): + m = metrics[grp] + lines.append(f"--- {GROUP_LABELS[grp]} ---") + lines.append(f" Food collected : {m['final_food']:.0f}") + lines.append(f" Food rate : {m['food_rate']:.2f} /tick") + lines.append(f" Energy spent : {m['energy_spent']:.1f}") + lines.append(f" Efficiency : {m['efficiency']:.3f} food/energy") + lines.append("") + + # Pairwise comparisons + lines.append("--- Pairwise Comparison ---") + for key, eff in effects.items(): + g1, g2 = key.split("_vs_") + lines.append(f" {g1.upper()} vs {g2.upper()}:") + lines.append(f" Food difference : {eff['food_diff']:+.0f}") + lines.append(f" Efficiency difference : {eff['efficiency_diff']:+.3f}") + lines.append(f" Energy difference : {eff['energy_diff']:+.1f}") + lines.append("") + + # Verdict + ranked = sorted(metrics.items(), key=lambda kv: kv[1]["final_food"], reverse=True) + winner = ranked[0][0] + lines.append(f"Best performing ordering: {GROUP_LABELS[winner]}") + lines.append("=" * 60) + return "\n".join(lines) + + +def plot_comparison(df: pd.DataFrame, output_path: str) -> None: + """Generate a multi-panel comparison plot and save to output_path.""" + fig, axes = plt.subplots(1, 2, figsize=(12, 5)) + + # Food over time + ax = axes[0] + ax.plot(df["tick"], df["group_a_food"], "r-", label="A: sense→move→share") + ax.plot(df["tick"], df["group_b_food"], "b-", label="B: share→sense→move") + ax.plot(df["tick"], df["group_c_food"], "g-", label="C: move→share→sense") + ax.set_xlabel("Tick") + ax.set_ylabel("Cumulative Food Collected") + ax.set_title("Food Collection by Rule Ordering") + ax.legend(fontsize=8) + ax.grid(True, alpha=0.3) + + # Energy over time + ax = axes[1] + ax.plot(df["tick"], df["group_a_energy"], "r-", label="A: sense→move→share") + ax.plot(df["tick"], df["group_b_energy"], "b-", label="B: share→sense→move") + ax.plot(df["tick"], df["group_c_energy"], "g-", label="C: move→share→sense") + ax.set_xlabel("Tick") + ax.set_ylabel("Average Energy") + ax.set_title("Energy Expenditure by Rule Ordering") + ax.legend(fontsize=8) + ax.grid(True, alpha=0.3) + + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def main(argv=None): + """CLI entry point.""" + parser = argparse.ArgumentParser( + description="Analyze ordering-matters simulation output." + ) + parser.add_argument("csv_file", help="Path to simulation CSV export") + parser.add_argument("--plot", metavar="FILE", help="Save comparison plot to FILE") + args = parser.parse_args(argv) + + df = load_simulation_data(args.csv_file) + metrics = compute_group_metrics(df) + effects = compute_effect_size(metrics) + report = generate_report(metrics, effects) + print(report) + + if args.plot: + plot_comparison(df, args.plot) + print(f"\nPlot saved to {args.plot}") + + +if __name__ == "__main__": + main() diff --git a/demos/ordering-matters/config b/demos/ordering-matters/config new file mode 100644 index 0000000..62e1b68 --- /dev/null +++ b/demos/ordering-matters/config @@ -0,0 +1,45 @@ +# Ordering-Matters Demo — LLM Configuration +# Choose ONE provider by uncommenting the relevant section + +# ============================================ +# OPTION 1: Ollama (Local, FREE, No API Key) +# ============================================ +# Best for: Testing, privacy, no costs +# Setup: Install Ollama from https://ollama.ai +# Then run: ollama pull llama3.2 +#provider=ollama +#model=llama3.2:latest +#base_url=http://localhost:11434 +#temperature=0.4 +#max_tokens=300 +#timeout_seconds=60 + +# ============================================ +# OPTION 2: OpenAI (Cloud, Requires API Key) +# ============================================ +provider=openai +api_key=YOUR_OPENAI_API_KEY_HERE +model=gpt-4o-mini +temperature=0.4 +max_tokens=300 +timeout_seconds=30 + +# ============================================ +# OPTION 3: Anthropic Claude (Cloud, API Key) +# ============================================ +#provider=anthropic +#api_key=YOUR_ANTHROPIC_API_KEY_HERE +#model=claude-3-5-sonnet-20241022 +#temperature=0.4 +#max_tokens=300 +#timeout_seconds=30 + +# ============================================ +# OPTION 4: Google Gemini (Cloud, API Key) +# ============================================ +#provider=gemini +#api_key=YOUR_GEMINI_API_KEY_HERE +#model=gemini-1.5-flash +#temperature=0.4 +#max_tokens=300 +#timeout_seconds=30 diff --git a/demos/ordering-matters/ordering-matters.nlogo b/demos/ordering-matters/ordering-matters.nlogo new file mode 100644 index 0000000..a3d8e90 --- /dev/null +++ b/demos/ordering-matters/ordering-matters.nlogo @@ -0,0 +1,728 @@ +;; ABOUTME: Demonstrates that rule execution ORDER affects emergent agent behavior. +;; ABOUTME: Three groups apply identical rules (sense/move/share) in different orders. + +extensions [llm] + +globals [ + group-a-food ;; cumulative food collected by Group A + group-b-food ;; cumulative food collected by Group B + group-c-food ;; cumulative food collected by Group C + tick-data ;; list of per-tick metric rows for CSV export +] + +turtles-own [ + group ;; "A", "B", or "C" + sensed-food-x ;; x coordinate of nearest sensed food + sensed-food-y ;; y coordinate of nearest sensed food + has-sensed? ;; true if food was detected this tick + shared-info ;; text received via LLM communication + food-collected ;; individual food counter + agent-energy ;; energy remaining (decreases with movement) +] + +patches-own [ + has-food? ;; whether this patch currently has food +] + +;; ───────────────────────────────────────────────────────────── +;; SETUP +;; ───────────────────────────────────────────────────────────── + +to setup + clear-all + + if use-llm? [ + llm:load-config "demos/ordering-matters/config" + ] + + ;; Scatter food patches + ask n-of food-count patches [ + set has-food? true + set pcolor green + ] + + ;; Divide agents evenly into three groups + let group-size floor (num-agents / 3) + + ;; Group A: sense → move → share (red circles) + create-turtles group-size [ + set group "A" + set color red + set shape "circle" + setxy random-xcor random-ycor + init-agent + ] + + ;; Group B: share → sense → move (blue squares) + create-turtles group-size [ + set group "B" + set color blue + set shape "square" + setxy random-xcor random-ycor + init-agent + ] + + ;; Group C: move → share → sense (lime triangles) + create-turtles group-size [ + set group "C" + set color green + 2 + set shape "triangle" + setxy random-xcor random-ycor + init-agent + ] + + set tick-data [] + reset-ticks +end + +to init-agent + set sensed-food-x 0 + set sensed-food-y 0 + set has-sensed? false + set shared-info "none" + set food-collected 0 + set agent-energy 100 +end + +;; ───────────────────────────────────────────────────────────── +;; MAIN LOOP +;; ───────────────────────────────────────────────────────────── + +to go + if not any? patches with [has-food?] and not respawn-food? [ stop ] + + ;; Each group applies the SAME three rules in a DIFFERENT order. + ;; This is the core demonstration: ordering matters. + + ;; Group A: sense → move → share + ask turtles with [group = "A"] [ + rule-sense + rule-move + rule-share + try-collect-food + ] + + ;; Group B: share → sense → move + ask turtles with [group = "B"] [ + rule-share + rule-sense + rule-move + try-collect-food + ] + + ;; Group C: move → share → sense + ask turtles with [group = "C"] [ + rule-move + rule-share + rule-sense + try-collect-food + ] + + update-metrics + + ;; Periodic food respawn + if respawn-food? and ticks mod respawn-interval = 0 and ticks > 0 [ + let available patches with [not has-food?] + let spawn-count min (list food-count count available) + ask n-of spawn-count available [ + set has-food? true + set pcolor green + ] + ] + + tick +end + +;; ───────────────────────────────────────────────────────────── +;; THE THREE RULES (identical logic, order varies by group) +;; ───────────────────────────────────────────────────────────── + +to rule-sense + ;; Detect nearest food within sensor-range + let nearby-food patches in-radius sensor-range with [has-food?] + ifelse any? nearby-food [ + let target min-one-of nearby-food [distance myself] + set sensed-food-x [pxcor] of target + set sensed-food-y [pycor] of target + set has-sensed? true + ] [ + set has-sensed? false + ] +end + +to rule-move + ;; Move toward food if sensed, use shared info if available, else wander + ifelse has-sensed? [ + facexy sensed-food-x sensed-food-y + fd min (list speed distance (patch sensed-food-x sensed-food-y)) + ] [ + ifelse shared-info != "none" and shared-info != "" [ + let angle extract-heading shared-info + if angle != -1 [ set heading angle ] + fd speed + ] [ + rt random 90 - 45 + fd speed + ] + ] + set agent-energy agent-energy - 1 +end + +to rule-share + ;; Communicate food knowledge with nearby agents via LLM (or simple mode) + if ticks mod share-interval != 0 [ stop ] + + let neighbors other turtles in-radius comm-range + if not any? neighbors [ stop ] + + let nearest min-one-of neighbors [distance myself] + + ifelse use-llm? [ + ;; Build context message + let my-info build-info-string + let prompt (word "You are a foraging agent. " my-info + " Advise a nearby agent in one short sentence. " + "Include a compass heading (0-360) if you know where food is.") + let response llm:chat prompt + ask nearest [ set shared-info response ] + ] [ + ;; Simple mode: share raw coordinates + ifelse has-sensed? [ + ask nearest [ + set shared-info (word sensed-food-x " " sensed-food-y) + ] + ] [ + ask nearest [ set shared-info "none" ] + ] + ] +end + +;; ───────────────────────────────────────────────────────────── +;; HELPERS +;; ───────────────────────────────────────────────────────────── + +to try-collect-food + if [has-food?] of patch-here [ + set food-collected food-collected + 1 + ask patch-here [ + set has-food? false + set pcolor black + ] + ] +end + +to-report build-info-string + ifelse has-sensed? [ + report (word "I see food at (" sensed-food-x "," sensed-food-y + "). I am at (" round xcor "," round ycor ").") + ] [ + report (word "I see no food nearby. I am at (" + round xcor "," round ycor ").") + ] +end + +to-report extract-heading [text] + ;; Parse a numeric heading (0-360) from text + if text = "" or text = "none" [ report -1 ] + let result -1 + let i 0 + while [i < length text - 1] [ + let ch item i text + if is-digit? ch [ + let num-str (word ch) + let j i + 1 + while [j < length text and is-digit? item j text] [ + set num-str (word num-str item j text) + set j j + 1 + ] + let num read-from-string num-str + if num >= 0 and num <= 360 [ set result num ] + ] + set i i + 1 + ] + report result +end + +to-report is-digit? [ch] + report member? (word ch) ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"] +end + +;; ───────────────────────────────────────────────────────────── +;; METRICS & EXPORT +;; ───────────────────────────────────────────────────────────── + +to update-metrics + set group-a-food sum [food-collected] of turtles with [group = "A"] + set group-b-food sum [food-collected] of turtles with [group = "B"] + set group-c-food sum [food-collected] of turtles with [group = "C"] + + let row (list ticks group-a-food group-b-food group-c-food + safe-mean "A" safe-mean "B" safe-mean "C") + set tick-data lput row tick-data +end + +to-report safe-mean [grp] + let agents turtles with [group = grp] + ifelse any? agents [ report mean [agent-energy] of agents ] [ report 0 ] +end + +to export-data + let filename "ordering-matters-output.csv" + carefully [ + file-open filename + file-print "tick,group_a_food,group_b_food,group_c_food,group_a_energy,group_b_energy,group_c_energy" + foreach tick-data [ row -> + file-print (word item 0 row "," item 1 row "," item 2 row "," + item 3 row "," item 4 row "," item 5 row "," item 6 row) + ] + file-close + output-print (word "Exported " length tick-data " rows to " filename) + ] [ + output-print (word "Export failed: " error-message) + ] +end + +to infer-rules + ;; Use LLM template to infer rule orderings from behavioral data + if not use-llm? [ + output-print "Enable use-llm? to run rule inference." + stop + ] + + let a-data (word "Food: " group-a-food + ", Energy: " precision safe-mean "A" 1 + ", Cluster: " precision cluster-spread "A" 2) + let b-data (word "Food: " group-b-food + ", Energy: " precision safe-mean "B" 1 + ", Cluster: " precision cluster-spread "B" 2) + let c-data (word "Food: " group-c-food + ", Energy: " precision safe-mean "C" 1 + ", Cluster: " precision cluster-spread "C" 2) + + let result llm:chat-with-template + "demos/ordering-matters/rule-inference-template.yaml" + (list + (list "tick_count" (word ticks)) + (list "food_density" (word food-count)) + (list "group_a_data" a-data) + (list "group_b_data" b-data) + (list "group_c_data" c-data)) + + output-print "=== Rule Ordering Inference ===" + output-print result +end + +to-report cluster-spread [grp] + ;; Average pairwise distance between agents in a group (lower = more clustered) + let agents turtles with [group = grp] + if count agents < 2 [ report 0 ] + let total-dist 0 + let pairs 0 + ask agents [ + ask other agents [ + set total-dist total-dist + distance myself + set pairs pairs + 1 + ] + ] + ifelse pairs > 0 [ report total-dist / pairs ] [ report 0 ] +end +@#$#@#$#@ +GRAPHICS-WINDOW +380 +10 +818 +449 +-1 +-1 +13.0 +1 +10 +1 +1 +1 +0 +1 +1 +1 +-16 +16 +-16 +16 +1 +1 +1 +ticks +30.0 + +BUTTON +10 +45 +80 +78 +NIL +setup +NIL +1 +T +OBSERVER +NIL +NIL +NIL +NIL +1 + +BUTTON +90 +45 +155 +78 +go +go +NIL +1 +T +OBSERVER +NIL +NIL +NIL +NIL +1 + +BUTTON +165 +45 +255 +78 +go-forever +go +T +1 +T +OBSERVER +NIL +NIL +NIL +NIL +1 + +SLIDER +10 +10 +180 +43 +num-agents +num-agents +3 +60 +18.0 +3 +1 +NIL +HORIZONTAL + +SLIDER +190 +10 +370 +43 +food-count +food-count +10 +200 +80.0 +10 +1 +NIL +HORIZONTAL + +SLIDER +10 +85 +180 +118 +sensor-range +sensor-range +1 +10 +4.0 +1 +1 +NIL +HORIZONTAL + +SLIDER +190 +85 +370 +118 +speed +speed +0.5 +3 +1.0 +0.5 +1 +NIL +HORIZONTAL + +SLIDER +10 +125 +180 +158 +comm-range +comm-range +1 +15 +6.0 +1 +1 +NIL +HORIZONTAL + +SLIDER +190 +125 +370 +158 +share-interval +share-interval +1 +20 +5.0 +1 +1 +ticks +HORIZONTAL + +SWITCH +10 +165 +130 +198 +use-llm? +use-llm? +1 +1 +-1000 + +SWITCH +140 +165 +280 +198 +respawn-food? +respawn-food? +0 +1 +-1000 + +SLIDER +10 +205 +180 +238 +respawn-interval +respawn-interval +5 +50 +20.0 +5 +1 +ticks +HORIZONTAL + +MONITOR +10 +250 +130 +295 +A food +group-a-food +0 +1 +11 + +MONITOR +140 +250 +260 +295 +B food +group-b-food +0 +1 +11 + +MONITOR +270 +250 +370 +295 +C food +group-c-food +0 +1 +11 + +BUTTON +10 +305 +130 +338 +export-data +export-data +NIL +1 +T +OBSERVER +NIL +NIL +NIL +NIL +1 + +BUTTON +140 +305 +260 +338 +infer-rules +infer-rules +NIL +1 +T +OBSERVER +NIL +NIL +NIL +NIL +1 + +PLOT +10 +345 +370 +520 +Food Collection +tick +food +0.0 +10.0 +0.0 +10.0 +true +true +"" "" +PENS +"A: sense-move-share" 1.0 0 -2674135 true "" "plot group-a-food" +"B: share-sense-move" 1.0 0 -13345367 true "" "plot group-b-food" +"C: move-share-sense" 1.0 0 -8732573 true "" "plot group-c-food" + +OUTPUT +10 +530 +370 +670 +12 + +@#$#@#$#@ +## WHAT IS IT? + +This model demonstrates that the **order** in which agents execute identical rules produces different emergent behaviors. Three groups of foraging agents each apply the same three rules — SENSE, MOVE, and SHARE — but in different sequences. + +## HOW IT WORKS + +Each tick, agents execute three rules: + +- **SENSE**: Detect food patches within `sensor-range` +- **MOVE**: Navigate toward food (if sensed), follow shared advice, or wander +- **SHARE**: Communicate food locations with nearby agents (via LLM or simple coordinates) + +The groups differ only in execution order: +- **Group A** (red): sense → move → share +- **Group B** (blue): share → sense → move +- **Group C** (green): move → share → sense + +## HOW TO USE IT + +1. Configure `config.txt` with your LLM provider (or disable `use-llm?`) +2. Click **setup** to initialize agents and food +3. Click **go-forever** to run the simulation +4. Compare food collection rates across groups in the plot +5. Click **export-data** to save CSV for `analysis.py` +6. Click **infer-rules** to have the LLM analyze behavior patterns + +## THINGS TO NOTICE + +- Group A (sense-first) typically finds food most efficiently +- Group B (share-first) tends to cluster, sharing outdated information +- Group C (move-first) wastes energy moving before sensing + +## THINGS TO TRY + +- Toggle `use-llm?` to compare LLM communication vs. simple coordinate sharing +- Vary `sensor-range` and `comm-range` to change the relative value of sensing vs sharing +- Increase `share-interval` to reduce communication frequency + +## EXTENDING THE MODEL + +- Add a fourth ordering (e.g., sense → share → move) +- Let agents evolve their ordering over time +- Add obstacles or predators to make ordering trade-offs sharper + +## CREDITS AND REFERENCES + +Part of the NetLogo LLM Extension demo collection. +@#$#@#$#@ +default +true +0 +Polygon -7500403 true true 150 5 40 250 150 205 260 250 + +circle +false +0 +Circle -7500403 true true 0 0 300 + +square +false +0 +Rectangle -7500403 true true 30 30 270 270 + +triangle +false +0 +Polygon -7500403 true true 150 30 15 255 285 255 +@#$#@#$#@ +NetLogo 6.4.0 +@#$#@#$#@ +@#$#@#$#@ +@#$#@#$#@ + + +setup +go + +group-a-food +group-b-food +group-c-food +safe-mean "A" +safe-mean "B" +safe-mean "C" + + +@#$#@#$#@ +@#$#@#$#@ +default +0.0 +-0.2 0 0.0 1.0 +0.0 1 1.0 0.0 +0.2 0 0.0 1.0 +link direction +true +0 +Line -7500403 true 0 150 0 0 150 300 +@#$#@#$#@ +1 +@#$#@#$#@ diff --git a/demos/ordering-matters/rule-inference-template.yaml b/demos/ordering-matters/rule-inference-template.yaml new file mode 100644 index 0000000..01ff2e9 --- /dev/null +++ b/demos/ordering-matters/rule-inference-template.yaml @@ -0,0 +1,23 @@ +system: "You are an expert in agent-based modeling who analyzes emergent behavior patterns to infer underlying rule structures." +template: | + Three groups of foraging agents (A, B, C) each apply the same three rules + (SENSE, MOVE, SHARE) but in different orders. Analyze the behavioral data + below and infer which ordering each group is likely using. + + Possible orderings: + - sense -> move -> share (react to own observations first) + - share -> sense -> move (communicate before acting) + - move -> share -> sense (act impulsively, reflect later) + + Simulation data after {tick_count} ticks with {food_density} food patches: + + Group A: {group_a_data} + Group B: {group_b_data} + Group C: {group_c_data} + + For each group, reason about: + 1. Does high food collection suggest sensing before moving? + 2. Does clustering suggest sharing before sensing? + 3. Does high energy cost suggest moving before sensing? + + Provide your inference for each group's rule ordering and confidence level. diff --git a/demos/ordering-matters/tests/__init__.py b/demos/ordering-matters/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/demos/ordering-matters/tests/test_analysis.py b/demos/ordering-matters/tests/test_analysis.py new file mode 100644 index 0000000..be2e042 --- /dev/null +++ b/demos/ordering-matters/tests/test_analysis.py @@ -0,0 +1,167 @@ +# ABOUTME: Unit tests for ordering-matters analysis script. +# ABOUTME: Validates CSV parsing, metric computation, effect size, and report generation. + +import csv +import os +import tempfile + +import pytest + +# Import module under test (parent directory) +import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +import analysis + + +@pytest.fixture +def sample_csv(tmp_path): + """Create a minimal simulation output CSV for testing.""" + filepath = tmp_path / "sim_output.csv" + rows = [ + ["tick", "group_a_food", "group_b_food", "group_c_food", + "group_a_energy", "group_b_energy", "group_c_energy"], + [0, 0, 0, 0, 100.0, 100.0, 100.0], + [1, 2, 0, 1, 98.5, 99.0, 97.0], + [2, 5, 1, 2, 96.0, 98.0, 94.5], + [3, 9, 3, 4, 93.0, 96.5, 92.0], + [4, 14, 5, 7, 90.0, 95.0, 89.5], + [5, 20, 8, 10, 87.0, 93.5, 87.0], + ] + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + writer.writerows(rows) + return str(filepath) + + +@pytest.fixture +def sample_df(sample_csv): + """Load the sample CSV as a DataFrame.""" + return analysis.load_simulation_data(sample_csv) + + +# ── load_simulation_data ───────────────────────────────────── + +class TestLoadSimulationData: + def test_loads_valid_csv(self, sample_csv): + df = analysis.load_simulation_data(sample_csv) + assert len(df) == 6 + assert "tick" in df.columns + assert "group_a_food" in df.columns + + def test_raises_on_missing_file(self): + with pytest.raises(FileNotFoundError): + analysis.load_simulation_data("/nonexistent/file.csv") + + def test_column_types_are_numeric(self, sample_csv): + df = analysis.load_simulation_data(sample_csv) + assert df["tick"].dtype in ("int64", "float64") + assert df["group_a_food"].dtype in ("int64", "float64") + assert df["group_a_energy"].dtype == "float64" + + def test_raises_on_missing_columns(self, tmp_path): + filepath = tmp_path / "bad.csv" + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["tick", "unrelated"]) + writer.writerow([0, 1]) + with pytest.raises(ValueError, match="Missing required columns"): + analysis.load_simulation_data(str(filepath)) + + +# ── compute_group_metrics ──────────────────────────────────── + +class TestComputeGroupMetrics: + def test_returns_all_groups(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + for grp in ("a", "b", "c"): + assert grp in metrics + + def test_final_food_values(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + assert metrics["a"]["final_food"] == 20 + assert metrics["b"]["final_food"] == 8 + assert metrics["c"]["final_food"] == 10 + + def test_food_rate_positive(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + for grp in ("a", "b", "c"): + assert metrics[grp]["food_rate"] > 0 + + def test_energy_efficiency(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + # efficiency = food_collected / energy_spent + assert metrics["a"]["efficiency"] == pytest.approx(20.0 / 13.0, rel=0.01) + + def test_single_tick_data(self, tmp_path): + filepath = tmp_path / "single.csv" + rows = [ + ["tick", "group_a_food", "group_b_food", "group_c_food", + "group_a_energy", "group_b_energy", "group_c_energy"], + [0, 5, 3, 4, 95.0, 97.0, 96.0], + ] + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + writer.writerows(rows) + df = analysis.load_simulation_data(str(filepath)) + metrics = analysis.compute_group_metrics(df) + assert metrics["a"]["final_food"] == 5 + + +# ── compute_effect_size ────────────────────────────────────── + +class TestComputeEffectSize: + def test_returns_pairwise_comparisons(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + effects = analysis.compute_effect_size(metrics) + assert "a_vs_b" in effects + assert "a_vs_c" in effects + assert "b_vs_c" in effects + + def test_effect_direction(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + effects = analysis.compute_effect_size(metrics) + # Group A collected more than B, so a_vs_b should be positive + assert effects["a_vs_b"]["food_diff"] > 0 + + def test_identical_groups_zero_effect(self): + metrics = { + "a": {"final_food": 10, "food_rate": 2.0, "efficiency": 1.0, + "final_energy": 90.0, "energy_spent": 10.0}, + "b": {"final_food": 10, "food_rate": 2.0, "efficiency": 1.0, + "final_energy": 90.0, "energy_spent": 10.0}, + "c": {"final_food": 10, "food_rate": 2.0, "efficiency": 1.0, + "final_energy": 90.0, "energy_spent": 10.0}, + } + effects = analysis.compute_effect_size(metrics) + assert effects["a_vs_b"]["food_diff"] == 0 + assert effects["a_vs_b"]["efficiency_diff"] == pytest.approx(0.0) + + +# ── generate_report ────────────────────────────────────────── + +class TestGenerateReport: + def test_report_contains_all_sections(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + effects = analysis.compute_effect_size(metrics) + report = analysis.generate_report(metrics, effects) + assert "Group A" in report + assert "Group B" in report + assert "Group C" in report + assert "Effect" in report or "Comparison" in report + + def test_report_is_string(self, sample_df): + metrics = analysis.compute_group_metrics(sample_df) + effects = analysis.compute_effect_size(metrics) + report = analysis.generate_report(metrics, effects) + assert isinstance(report, str) + assert len(report) > 100 + + +# ── plot_comparison (smoke test) ───────────────────────────── + +class TestPlotComparison: + def test_creates_output_file(self, sample_df, tmp_path): + outpath = str(tmp_path / "plot.png") + analysis.plot_comparison(sample_df, outpath) + assert os.path.exists(outpath) + assert os.path.getsize(outpath) > 0 From 9289d89fdc1557f4f652abd9ad4a647594c63fe5 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:19 -0600 Subject: [PATCH 2/9] feat(ordering-matters): rebuild NetLogo demo around trajectory ordering inference --- demos/ordering-matters/ordering-matters.nlogo | 678 ++++++++++-------- 1 file changed, 389 insertions(+), 289 deletions(-) diff --git a/demos/ordering-matters/ordering-matters.nlogo b/demos/ordering-matters/ordering-matters.nlogo index a3d8e90..5b7f241 100644 --- a/demos/ordering-matters/ordering-matters.nlogo +++ b/demos/ordering-matters/ordering-matters.nlogo @@ -1,206 +1,127 @@ -;; ABOUTME: Demonstrates that rule execution ORDER affects emergent agent behavior. -;; ABOUTME: Three groups apply identical rules (sense/move/share) in different orders. +;; ABOUTME: Tests whether LLM rule learning depends on trajectory ordering. +;; ABOUTME: The same agent trajectories are presented as forward, reversed, and shuffled. extensions [llm] globals [ - group-a-food ;; cumulative food collected by Group A - group-b-food ;; cumulative food collected by Group B - group-c-food ;; cumulative food collected by Group C - tick-data ;; list of per-tick metric rows for CSV export + ;; UI-linked metrics (kept names to match existing widgets) + group-a-food ;; forward ordering confidence (% if parsed) + group-b-food ;; reversed ordering confidence (% if parsed) + group-c-food ;; shuffled ordering confidence (% if parsed) + + food-collected-total ;; total food collected by all agents + run-label ;; short id for export files + + trajectory-log ;; list of [tick who x y heading energy food-total] + trajectory-forward ;; trajectory text in chronological order + trajectory-reversed ;; trajectory text reversed in time + trajectory-shuffled ;; trajectory text randomly shuffled + + inferred-forward ;; LLM inference text from forward trajectory order + inferred-reversed ;; LLM inference text from reversed trajectory order + inferred-shuffled ;; LLM inference text from shuffled trajectory order + + sample-size ;; max number of trajectory rows sent to prompt ] turtles-own [ - group ;; "A", "B", or "C" - sensed-food-x ;; x coordinate of nearest sensed food - sensed-food-y ;; y coordinate of nearest sensed food - has-sensed? ;; true if food was detected this tick - shared-info ;; text received via LLM communication - food-collected ;; individual food counter - agent-energy ;; energy remaining (decreases with movement) + agent-energy + food-collected ] patches-own [ - has-food? ;; whether this patch currently has food + has-food? ] -;; ───────────────────────────────────────────────────────────── -;; SETUP -;; ───────────────────────────────────────────────────────────── +;; ----------------------------------------------------------------------------- +;; Setup / Simulation +;; ----------------------------------------------------------------------------- to setup clear-all - if use-llm? [ - llm:load-config "demos/ordering-matters/config" - ] - - ;; Scatter food patches - ask n-of food-count patches [ - set has-food? true - set pcolor green + set run-label (word "run-" (100000 + random 900000)) + set sample-size 240 + + set group-a-food 0 + set group-b-food 0 + set group-c-food 0 + set food-collected-total 0 + + set trajectory-log [] + set trajectory-forward "" + set trajectory-reversed "" + set trajectory-shuffled "" + set inferred-forward "" + set inferred-reversed "" + set inferred-shuffled "" + + ask patches [ + set has-food? false + set pcolor black ] - ;; Divide agents evenly into three groups - let group-size floor (num-agents / 3) + seed-food - ;; Group A: sense → move → share (red circles) - create-turtles group-size [ - set group "A" - set color red + create-turtles num-agents [ set shape "circle" + set color orange + 2 + set size 1.1 setxy random-xcor random-ycor - init-agent + set heading random 360 + set agent-energy 100 + set food-collected 0 ] - ;; Group B: share → sense → move (blue squares) - create-turtles group-size [ - set group "B" - set color blue - set shape "square" - setxy random-xcor random-ycor - init-agent - ] - - ;; Group C: move → share → sense (lime triangles) - create-turtles group-size [ - set group "C" - set color green + 2 - set shape "triangle" - setxy random-xcor random-ycor - init-agent + if use-llm? [ + load-llm-config ] - set tick-data [] reset-ticks end -to init-agent - set sensed-food-x 0 - set sensed-food-y 0 - set has-sensed? false - set shared-info "none" - set food-collected 0 - set agent-energy 100 -end - -;; ───────────────────────────────────────────────────────────── -;; MAIN LOOP -;; ───────────────────────────────────────────────────────────── - to go - if not any? patches with [has-food?] and not respawn-food? [ stop ] - - ;; Each group applies the SAME three rules in a DIFFERENT order. - ;; This is the core demonstration: ordering matters. - - ;; Group A: sense → move → share - ask turtles with [group = "A"] [ - rule-sense - rule-move - rule-share + ask turtles [ + hidden-behavior try-collect-food ] - ;; Group B: share → sense → move - ask turtles with [group = "B"] [ - rule-share - rule-sense - rule-move - try-collect-food - ] + set food-collected-total sum [food-collected] of turtles - ;; Group C: move → share → sense - ask turtles with [group = "C"] [ - rule-move - rule-share - rule-sense - try-collect-food - ] - - update-metrics - - ;; Periodic food respawn - if respawn-food? and ticks mod respawn-interval = 0 and ticks > 0 [ - let available patches with [not has-food?] - let spawn-count min (list food-count count available) - ask n-of spawn-count available [ - set has-food? true - set pcolor green - ] + if respawn-food? and ticks > 0 and ticks mod respawn-interval = 0 [ + respawn-food ] + record-trajectories tick end -;; ───────────────────────────────────────────────────────────── -;; THE THREE RULES (identical logic, order varies by group) -;; ───────────────────────────────────────────────────────────── +to hidden-behavior + ;; Hidden policy that we want the LLM to infer from observed motion: + ;; 1) seek nearby food, 2) avoid crowding, 3) keep momentum with mild noise. -to rule-sense - ;; Detect nearest food within sensor-range let nearby-food patches in-radius sensor-range with [has-food?] - ifelse any? nearby-food [ - let target min-one-of nearby-food [distance myself] - set sensed-food-x [pxcor] of target - set sensed-food-y [pycor] of target - set has-sensed? true - ] [ - set has-sensed? false + if any? nearby-food [ + face min-one-of nearby-food [distance myself] ] -end -to rule-move - ;; Move toward food if sensed, use shared info if available, else wander - ifelse has-sensed? [ - facexy sensed-food-x sensed-food-y - fd min (list speed distance (patch sensed-food-x sensed-food-y)) - ] [ - ifelse shared-info != "none" and shared-info != "" [ - let angle extract-heading shared-info - if angle != -1 [ set heading angle ] - fd speed - ] [ - rt random 90 - 45 - fd speed - ] + let crowd other turtles in-radius max (list 1 (comm-range / 2)) + if any? crowd [ + rt 25 + random 20 ] - set agent-energy agent-energy - 1 -end -to rule-share - ;; Communicate food knowledge with nearby agents via LLM (or simple mode) - if ticks mod share-interval != 0 [ stop ] + rt (random-float 16) - 8 + fd speed - let neighbors other turtles in-radius comm-range - if not any? neighbors [ stop ] + set agent-energy max (list 0 (agent-energy - (0.5 + speed * 0.4))) - let nearest min-one-of neighbors [distance myself] - - ifelse use-llm? [ - ;; Build context message - let my-info build-info-string - let prompt (word "You are a foraging agent. " my-info - " Advise a nearby agent in one short sentence. " - "Include a compass heading (0-360) if you know where food is.") - let response llm:chat prompt - ask nearest [ set shared-info response ] - ] [ - ;; Simple mode: share raw coordinates - ifelse has-sensed? [ - ask nearest [ - set shared-info (word sensed-food-x " " sensed-food-y) - ] - ] [ - ask nearest [ set shared-info "none" ] - ] + if agent-energy <= 0 [ + setxy random-xcor random-ycor + set heading random 360 + set agent-energy 45 ] end -;; ───────────────────────────────────────────────────────────── -;; HELPERS -;; ───────────────────────────────────────────────────────────── - to try-collect-food if [has-food?] of patch-here [ set food-collected food-collected + 1 @@ -211,120 +132,310 @@ to try-collect-food ] end -to-report build-info-string - ifelse has-sensed? [ - report (word "I see food at (" sensed-food-x "," sensed-food-y - "). I am at (" round xcor "," round ycor ").") - ] [ - report (word "I see no food nearby. I am at (" - round xcor "," round ycor ").") +to seed-food + ask n-of food-count patches [ + set has-food? true + set pcolor green + 1 ] end -to-report extract-heading [text] - ;; Parse a numeric heading (0-360) from text - if text = "" or text = "none" [ report -1 ] - let result -1 - let i 0 - while [i < length text - 1] [ - let ch item i text - if is-digit? ch [ - let num-str (word ch) - let j i + 1 - while [j < length text and is-digit? item j text] [ - set num-str (word num-str item j text) - set j j + 1 - ] - let num read-from-string num-str - if num >= 0 and num <= 360 [ set result num ] +to respawn-food + let candidates patches with [not has-food?] + if any? candidates [ + ask n-of min (list food-count count candidates) candidates [ + set has-food? true + set pcolor green + 1 ] - set i i + 1 ] - report result end -to-report is-digit? [ch] - report member? (word ch) ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"] +to record-trajectories + foreach sort turtles [ t -> + ask t [ + set trajectory-log lput + (list ticks who precision xcor 3 precision ycor 3 precision heading 2 precision agent-energy 2 food-collected) + trajectory-log + ] + ] end -;; ───────────────────────────────────────────────────────────── -;; METRICS & EXPORT -;; ───────────────────────────────────────────────────────────── +;; ----------------------------------------------------------------------------- +;; Ordering Construction + Inference +;; ----------------------------------------------------------------------------- -to update-metrics - set group-a-food sum [food-collected] of turtles with [group = "A"] - set group-b-food sum [food-collected] of turtles with [group = "B"] - set group-c-food sum [food-collected] of turtles with [group = "C"] +to infer-rules + if ticks = 0 [ + output-print "Run the simulation first (click go-forever for ~100 ticks)." + stop + ] + + build-trajectory-orderings - let row (list ticks group-a-food group-b-food group-c-food - safe-mean "A" safe-mean "B" safe-mean "C") - set tick-data lput row tick-data + if not use-llm? [ + set inferred-forward "LLM disabled. Hidden rule: seek food, avoid crowding, preserve momentum." + set inferred-reversed inferred-forward + set inferred-shuffled inferred-forward + set group-a-food 0 + set group-b-food 0 + set group-c-food 0 + output-print "LLM disabled. Generated deterministic baseline text instead." + stop + ] + + let template-path resolve-template-path + + set inferred-forward run-inference "forward" trajectory-forward template-path + set inferred-reversed run-inference "reversed" trajectory-reversed template-path + set inferred-shuffled run-inference "shuffled" trajectory-shuffled template-path + + set group-a-food extract-confidence inferred-forward + set group-b-food extract-confidence inferred-reversed + set group-c-food extract-confidence inferred-shuffled + + output-print "=== Rule Inference By Ordering ===" + output-print (word "FORWARD : " inferred-forward) + output-print (word "REVERSED : " inferred-reversed) + output-print (word "SHUFFLED : " inferred-shuffled) end -to-report safe-mean [grp] - let agents turtles with [group = grp] - ifelse any? agents [ report mean [agent-energy] of agents ] [ report 0 ] +to build-trajectory-orderings + if empty? trajectory-log [ + set trajectory-forward "" + set trajectory-reversed "" + set trajectory-shuffled "" + stop + ] + + let forward sort-by + [[r1 r2] -> + ifelse-value (item 0 r1 = item 0 r2) + [item 1 r1 < item 1 r2] + [item 0 r1 < item 0 r2] + ] + trajectory-log + + let reversed reverse forward + + random-seed 1776 + let shuffled shuffle forward + + set trajectory-forward format-trajectory forward + set trajectory-reversed format-trajectory reversed + set trajectory-shuffled format-trajectory shuffled +end + +to-report format-trajectory [rows] + let cap min (list sample-size length rows) + let clipped sublist rows 0 cap + + let lines map [row -> + (word + "tick=" item 0 row + " id=" item 1 row + " x=" item 2 row + " y=" item 3 row + " heading=" item 4 row + " energy=" item 5 row + " food=" item 6 row) + ] clipped + + report join-lines lines +end + +to-report join-lines [lines] + if empty? lines [ report "" ] + let out item 0 lines + foreach but-first lines [line -> + set out (word out "\n" line) + ] + report out end +to-report run-inference [ordering-label trajectory-text template-path] + let params (list + (list "ordering_label" ordering-label) + (list "tick_count" (word ticks)) + (list "agent_count" (word count turtles)) + (list "food_count" (word food-count)) + (list "trajectory_text" trajectory-text) + ) + + let response "" + carefully [ + set response llm:chat-with-template template-path params + ] [ + set response (word "Inference failed: " error-message) + ] + report response +end + +;; ----------------------------------------------------------------------------- +;; Export + Utility +;; ----------------------------------------------------------------------------- + to export-data - let filename "ordering-matters-output.csv" + build-trajectory-orderings + export-trajectory-csv + export-inference-csv +end + +to export-trajectory-csv + let filename (word "results/" run-label "-trajectories.csv") + carefully [ file-open filename - file-print "tick,group_a_food,group_b_food,group_c_food,group_a_energy,group_b_energy,group_c_energy" - foreach tick-data [ row -> - file-print (word item 0 row "," item 1 row "," item 2 row "," - item 3 row "," item 4 row "," item 5 row "," item 6 row) - ] + file-print "ordering,tick,agent_id,x,y,heading,energy,food_collected" + + write-ordering-rows "forward" trajectory-forward + write-ordering-rows "reversed" trajectory-reversed + write-ordering-rows "shuffled" trajectory-shuffled + file-close - output-print (word "Exported " length tick-data " rows to " filename) + output-print (word "Exported trajectories to " filename) ] [ - output-print (word "Export failed: " error-message) + output-print (word "Trajectory export failed: " error-message) + carefully [ file-close ] [ ] ] end -to infer-rules - ;; Use LLM template to infer rule orderings from behavioral data - if not use-llm? [ - output-print "Enable use-llm? to run rule inference." - stop +to write-ordering-rows [ordering text-block] + if text-block = "" [ stop ] + let lines split-lines text-block + foreach lines [line -> + let tick-value parse-number-after "tick=" line + let id-value parse-number-after "id=" line + let x-value parse-number-after "x=" line + let y-value parse-number-after "y=" line + let h-value parse-number-after "heading=" line + let e-value parse-number-after "energy=" line + let f-value parse-number-after "food=" line + file-print (word ordering "," tick-value "," id-value "," x-value "," y-value "," h-value "," e-value "," f-value) + ] +end + +to export-inference-csv + let filename (word "results/" run-label "-inference.csv") + + carefully [ + file-open filename + file-print "ordering,inferred_rule,confidence" + file-print (word "forward,\"" sanitize-csv inferred-forward "\"," group-a-food) + file-print (word "reversed,\"" sanitize-csv inferred-reversed "\"," group-b-food) + file-print (word "shuffled,\"" sanitize-csv inferred-shuffled "\"," group-c-food) + file-close + output-print (word "Exported inferences to " filename) + ] [ + output-print (word "Inference export failed: " error-message) + carefully [ file-close ] [ ] + ] +end + +to load-llm-config + ifelse file-exists? "config.txt" [ + llm:load-config "config.txt" + ] [ + llm:load-config "demos/ordering-matters/config.txt" + ] +end + +to-report resolve-template-path + ifelse file-exists? "rule-inference-template.yaml" [ + report "rule-inference-template.yaml" + ] [ + report "demos/ordering-matters/rule-inference-template.yaml" + ] +end + +to-report extract-confidence [text] + ;; Attempts to parse a confidence value from model output. + ;; Accepts either 0-1 values or percentages. + let nums extract-all-numbers text + if empty? nums [ report 0 ] + + let best 0 + foreach nums [n -> + if n > 0 and n <= 1 [ set best max (list best (n * 100)) ] + if n > 1 and n <= 100 [ set best max (list best n) ] ] + report precision best 2 +end - let a-data (word "Food: " group-a-food - ", Energy: " precision safe-mean "A" 1 - ", Cluster: " precision cluster-spread "A" 2) - let b-data (word "Food: " group-b-food - ", Energy: " precision safe-mean "B" 1 - ", Cluster: " precision cluster-spread "B" 2) - let c-data (word "Food: " group-c-food - ", Energy: " precision safe-mean "C" 1 - ", Cluster: " precision cluster-spread "C" 2) - - let result llm:chat-with-template - "demos/ordering-matters/rule-inference-template.yaml" - (list - (list "tick_count" (word ticks)) - (list "food_density" (word food-count)) - (list "group_a_data" a-data) - (list "group_b_data" b-data) - (list "group_c_data" c-data)) - - output-print "=== Rule Ordering Inference ===" - output-print result +to-report sanitize-csv [text] + report replace-all text "\"" "''" end -to-report cluster-spread [grp] - ;; Average pairwise distance between agents in a group (lower = more clustered) - let agents turtles with [group = grp] - if count agents < 2 [ report 0 ] - let total-dist 0 - let pairs 0 - ask agents [ - ask other agents [ - set total-dist total-dist + distance myself - set pairs pairs + 1 +to-report split-lines [text] + if text = "" [ report [] ] + + let lines [] + let start 0 + let i 0 + while [i < length text] [ + if substring text i (i + 1) = "\n" [ + set lines lput (substring text start i) lines + set start i + 1 ] + set i i + 1 + ] + + if start <= length text [ + set lines lput (substring text start length text) lines + ] + + report filter [line -> line != ""] lines +end + +to-report parse-number-after [token line] + let p position token line + if p = false [ report 0 ] + + let start p + length token + let finish start + + while [finish < length line and not member? (substring line finish (finish + 1)) [" " ","]] [ + set finish finish + 1 ] - ifelse pairs > 0 [ report total-dist / pairs ] [ report 0 ] + + let s substring line start finish + if s = "" [ report 0 ] + report read-from-string s +end + +to-report extract-all-numbers [text] + let nums [] + let i 0 + while [i < length text] [ + let ch substring text i (i + 1) + if member? ch ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"] [ + let start i + let j i + while [j < length text and member? (substring text j (j + 1)) ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "."]] [ + set j j + 1 + ] + set nums lput (read-from-string (substring text start j)) nums + set i j + ] + set i i + 1 + ] + report nums +end + +to-report replace-all [text target replacement] + if text = "" [ report "" ] + let p position target text + if p = false [ report text ] + report (word + (substring text 0 p) + replacement + (replace-all (substring text (p + length target) (length text)) target replacement)) +end + +;; Reporter retained for existing BehaviorSpace metric entries. +to-report safe-mean [ordering-code] + if ordering-code = "A" [ report group-a-food ] + if ordering-code = "B" [ report group-b-food ] + if ordering-code = "C" [ report group-c-food ] + report 0 end @#$#@#$#@ GRAPHICS-WINDOW @@ -537,9 +648,9 @@ MONITOR 250 130 295 -A food +forward conf group-a-food -0 +2 1 11 @@ -548,9 +659,9 @@ MONITOR 250 260 295 -B food +reversed conf group-b-food -0 +2 1 11 @@ -559,9 +670,9 @@ MONITOR 250 370 295 -C food +shuffled conf group-c-food -0 +2 1 11 @@ -604,9 +715,9 @@ PLOT 345 370 520 -Food Collection +Inference Confidence by Ordering tick -food +confidence 0.0 10.0 0.0 @@ -615,9 +726,9 @@ true true "" "" PENS -"A: sense-move-share" 1.0 0 -2674135 true "" "plot group-a-food" -"B: share-sense-move" 1.0 0 -13345367 true "" "plot group-b-food" -"C: move-share-sense" 1.0 0 -8732573 true "" "plot group-c-food" +"forward" 1.0 0 -2674135 true "" "plot group-a-food" +"reversed" 1.0 0 -13345367 true "" "plot group-b-food" +"shuffled" 1.0 0 -8732573 true "" "plot group-c-food" OUTPUT 10 @@ -629,51 +740,43 @@ OUTPUT @#$#@#$#@ ## WHAT IS IT? -This model demonstrates that the **order** in which agents execute identical rules produces different emergent behaviors. Three groups of foraging agents each apply the same three rules — SENSE, MOVE, and SHARE — but in different sequences. - -## HOW IT WORKS - -Each tick, agents execute three rules: +This demo tests whether a language model learns different behavioral rules from the **same trajectory data** when the rows are presented in different orders: -- **SENSE**: Detect food patches within `sensor-range` -- **MOVE**: Navigate toward food (if sensed), follow shared advice, or wander -- **SHARE**: Communicate food locations with nearby agents (via LLM or simple coordinates) +- forward (chronological) +- reversed (time reversed) +- shuffled (random permutation) -The groups differ only in execution order: -- **Group A** (red): sense → move → share -- **Group B** (blue): share → sense → move -- **Group C** (green): move → share → sense +## HOW IT WORKS -## HOW TO USE IT +1. Agents follow one hidden policy while foraging: +- seek nearby food +- avoid local crowding +- preserve momentum with small heading noise -1. Configure `config.txt` with your LLM provider (or disable `use-llm?`) -2. Click **setup** to initialize agents and food -3. Click **go-forever** to run the simulation -4. Compare food collection rates across groups in the plot -5. Click **export-data** to save CSV for `analysis.py` -6. Click **infer-rules** to have the LLM analyze behavior patterns +2. The model records trajectories at each tick. -## THINGS TO NOTICE +3. `infer-rules` serializes the trajectory log into three orderings and prompts an LLM with the same template for each ordering. -- Group A (sense-first) typically finds food most efficiently -- Group B (share-first) tends to cluster, sharing outdated information -- Group C (move-first) wastes energy moving before sensing +4. `export-data` writes: +- `results/-trajectories.csv` +- `results/-inference.csv` -## THINGS TO TRY +## HOW TO USE IT -- Toggle `use-llm?` to compare LLM communication vs. simple coordinate sharing -- Vary `sensor-range` and `comm-range` to change the relative value of sensing vs sharing -- Increase `share-interval` to reduce communication frequency +1. Configure `config.txt` (or disable `use-llm?`) +2. Click `setup` +3. Run `go-forever` for 100-200 ticks +4. Click `infer-rules` +5. Click `export-data` +6. Analyze exports with `analysis.py` -## EXTENDING THE MODEL +## WHY IT MATTERS -- Add a fourth ordering (e.g., sense → share → move) -- Let agents evolve their ordering over time -- Add obstacles or predators to make ordering trade-offs sharper +If inferred rules differ strongly between forward/reversed/shuffled views, rule learning is order-sensitive. This is a practical failure mode when using LLMs for inverse behavior modeling. -## CREDITS AND REFERENCES +## CREDITS -Part of the NetLogo LLM Extension demo collection. +Part of the AgentSwarm NetLogo LLM demos. @#$#@#$#@ default true @@ -700,16 +803,13 @@ NetLogo 6.4.0 @#$#@#$#@ @#$#@#$#@ - + setup -go - +repeat 150 [go] group-a-food group-b-food group-c-food -safe-mean "A" -safe-mean "B" -safe-mean "C" +food-collected-total @#$#@#$#@ From 786c5312d4fa67ee1a3bcece20c359069b3016cc Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:23 -0600 Subject: [PATCH 3/9] feat(prompt): add ordering-aware rule inference template --- .../rule-inference-template.yaml | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/demos/ordering-matters/rule-inference-template.yaml b/demos/ordering-matters/rule-inference-template.yaml index 01ff2e9..cddde51 100644 --- a/demos/ordering-matters/rule-inference-template.yaml +++ b/demos/ordering-matters/rule-inference-template.yaml @@ -1,23 +1,26 @@ -system: "You are an expert in agent-based modeling who analyzes emergent behavior patterns to infer underlying rule structures." +system: | + You are an expert in inverse behavior modeling. + Infer an agent policy from observed trajectories. + template: | - Three groups of foraging agents (A, B, C) each apply the same three rules - (SENSE, MOVE, SHARE) but in different orders. Analyze the behavioral data - below and infer which ordering each group is likely using. + You are given trajectory rows from a 2D foraging simulation. - Possible orderings: - - sense -> move -> share (react to own observations first) - - share -> sense -> move (communicate before acting) - - move -> share -> sense (act impulsively, reflect later) + Ordering label: {ordering_label} + Ticks observed: {tick_count} + Agent count: {agent_count} + Initial food patches: {food_count} - Simulation data after {tick_count} ticks with {food_density} food patches: + Trajectory rows: + {trajectory_text} - Group A: {group_a_data} - Group B: {group_b_data} - Group C: {group_c_data} + Task: + Infer the most likely behavioral rule set that generated these trajectories. - For each group, reason about: - 1. Does high food collection suggest sensing before moving? - 2. Does clustering suggest sharing before sensing? - 3. Does high energy cost suggest moving before sensing? + Return exactly these fields: + - inferred_rule: short plain-English rule (1-2 sentences) + - evidence: 2-4 bullet points tied to trajectory patterns + - confidence: a number from 0 to 100 - Provide your inference for each group's rule ordering and confidence level. + Constraints: + - Do not mention ordering artifacts unless needed as uncertainty. + - Focus on motion regularities (goal-seeking, avoidance, momentum, randomness). From 1d00e0df8ee4b7725ef623d84393a4069fec6f7a Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:31 -0600 Subject: [PATCH 4/9] chore(config): switch demo to explicit config.txt provider file --- demos/ordering-matters/config | 45 ------------------------------- demos/ordering-matters/config.txt | 34 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 45 deletions(-) delete mode 100644 demos/ordering-matters/config create mode 100644 demos/ordering-matters/config.txt diff --git a/demos/ordering-matters/config b/demos/ordering-matters/config deleted file mode 100644 index 62e1b68..0000000 --- a/demos/ordering-matters/config +++ /dev/null @@ -1,45 +0,0 @@ -# Ordering-Matters Demo — LLM Configuration -# Choose ONE provider by uncommenting the relevant section - -# ============================================ -# OPTION 1: Ollama (Local, FREE, No API Key) -# ============================================ -# Best for: Testing, privacy, no costs -# Setup: Install Ollama from https://ollama.ai -# Then run: ollama pull llama3.2 -#provider=ollama -#model=llama3.2:latest -#base_url=http://localhost:11434 -#temperature=0.4 -#max_tokens=300 -#timeout_seconds=60 - -# ============================================ -# OPTION 2: OpenAI (Cloud, Requires API Key) -# ============================================ -provider=openai -api_key=YOUR_OPENAI_API_KEY_HERE -model=gpt-4o-mini -temperature=0.4 -max_tokens=300 -timeout_seconds=30 - -# ============================================ -# OPTION 3: Anthropic Claude (Cloud, API Key) -# ============================================ -#provider=anthropic -#api_key=YOUR_ANTHROPIC_API_KEY_HERE -#model=claude-3-5-sonnet-20241022 -#temperature=0.4 -#max_tokens=300 -#timeout_seconds=30 - -# ============================================ -# OPTION 4: Google Gemini (Cloud, API Key) -# ============================================ -#provider=gemini -#api_key=YOUR_GEMINI_API_KEY_HERE -#model=gemini-1.5-flash -#temperature=0.4 -#max_tokens=300 -#timeout_seconds=30 diff --git a/demos/ordering-matters/config.txt b/demos/ordering-matters/config.txt new file mode 100644 index 0000000..e265774 --- /dev/null +++ b/demos/ordering-matters/config.txt @@ -0,0 +1,34 @@ +# Ordering Matters demo LLM config +# Use exactly one provider block. + +# --- OpenAI --- +provider=openai +api_key=YOUR_OPENAI_API_KEY_HERE +model=gpt-4o-mini +temperature=0.2 +max_tokens=400 +timeout_seconds=45 + +# --- Anthropic --- +#provider=anthropic +#api_key=YOUR_ANTHROPIC_API_KEY_HERE +#model=claude-3-5-sonnet-20241022 +#temperature=0.2 +#max_tokens=400 +#timeout_seconds=45 + +# --- Gemini --- +#provider=gemini +#api_key=YOUR_GEMINI_API_KEY_HERE +#model=gemini-1.5-flash +#temperature=0.2 +#max_tokens=400 +#timeout_seconds=45 + +# --- Ollama (local) --- +#provider=ollama +#model=llama3.2:latest +#base_url=http://localhost:11434 +#temperature=0.2 +#max_tokens=400 +#timeout_seconds=90 From 8d1f0d7bec88b4c04c845b8be351c75612c343f9 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:36 -0600 Subject: [PATCH 5/9] feat(analysis): score rule divergence across trajectory orderings --- demos/ordering-matters/analysis.py | 481 +++++++++++++++++++---------- 1 file changed, 326 insertions(+), 155 deletions(-) diff --git a/demos/ordering-matters/analysis.py b/demos/ordering-matters/analysis.py index 9983aa1..69c7884 100644 --- a/demos/ordering-matters/analysis.py +++ b/demos/ordering-matters/analysis.py @@ -1,188 +1,359 @@ -# ABOUTME: Analyzes ordering-matters simulation CSV output to quantify how -# ABOUTME: rule execution order affects emergent agent behavior across groups. - -""" -Ordering-Matters Analysis -========================= -Parses NetLogo simulation exports and computes per-group metrics, -pairwise effect sizes, and summary reports. Optionally generates -comparison plots. - -Usage: - python3 analysis.py [--plot output.png] +# ABOUTME: Quantifies whether inferred behavioral rules change when identical +# ABOUTME: trajectories are presented in forward, reversed, or shuffled order. + +"""Ordering Matters analysis. + +Given an inference CSV exported from `ordering-matters.nlogo`, this script: +1. Normalizes inferred rule text per ordering. +2. Computes pairwise rule similarity. +3. Produces an order-dependency score. +4. Optionally writes JSON summary and a plot. + +Expected CSV columns: +- ordering: forward | reversed | shuffled +- inferred_rule: free text +- confidence: optional numeric 0-100 + +Example: + python3 analysis.py results/sample-inference.csv \ + --plot results/rule-similarity.png \ + --json results/analysis-summary.json """ +from __future__ import annotations + import argparse -import sys +import csv +import json +import math +import re +from dataclasses import dataclass +from difflib import SequenceMatcher +from itertools import combinations from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + +EXPECTED_ORDERINGS = ("forward", "reversed", "shuffled") +REQUIRED_COLUMNS = {"ordering", "inferred_rule"} + +STOP_WORDS = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "in", + "into", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "their", + "then", + "this", + "to", + "with", +} -import pandas as pd -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt -REQUIRED_COLUMNS = { - "tick", "group_a_food", "group_b_food", "group_c_food", - "group_a_energy", "group_b_energy", "group_c_energy", -} +@dataclass(frozen=True) +class InferenceRow: + ordering: str + inferred_rule: str + confidence: Optional[float] = None -GROUP_LABELS = { - "a": "Group A (sense→move→share)", - "b": "Group B (share→sense→move)", - "c": "Group C (move→share→sense)", -} +@dataclass(frozen=True) +class PairwiseSimilarity: + left: str + right: str + jaccard: float + sequence: float + combined: float + + +def parse_confidence(value: str) -> Optional[float]: + if value is None: + return None + text = value.strip().replace("%", "") + if not text: + return None + try: + parsed = float(text) + except ValueError: + return None + if math.isnan(parsed): + return None + return parsed + + +def load_inference_data(path: str) -> List[InferenceRow]: + csv_path = Path(path) + if not csv_path.exists(): + raise FileNotFoundError(f"No such file: {path}") + + with csv_path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + if reader.fieldnames is None: + raise ValueError("CSV has no header row") + + missing = REQUIRED_COLUMNS - set(reader.fieldnames) + if missing: + raise ValueError(f"Missing required columns: {sorted(missing)}") + + rows: List[InferenceRow] = [] + for raw in reader: + ordering = (raw.get("ordering") or "").strip().lower() + rule = (raw.get("inferred_rule") or "").strip() + confidence = parse_confidence(raw.get("confidence") or "") + if not ordering or not rule: + continue + rows.append(InferenceRow(ordering=ordering, inferred_rule=rule, confidence=confidence)) + + if not rows: + raise ValueError("CSV contains no usable inference rows") + + return dedupe_by_ordering(rows) -def load_simulation_data(filepath: str) -> pd.DataFrame: - """Load and validate a simulation output CSV. - - Raises FileNotFoundError if path is invalid, ValueError if required - columns are absent. - """ - path = Path(filepath) - if not path.exists(): - raise FileNotFoundError(f"No such file: {filepath}") - - df = pd.read_csv(filepath) - missing = REQUIRED_COLUMNS - set(df.columns) - if missing: - raise ValueError(f"Missing required columns: {sorted(missing)}") - - for col in df.columns: - df[col] = pd.to_numeric(df[col]) - - return df - - -def compute_group_metrics(df: pd.DataFrame) -> dict: - """Compute per-group summary metrics from simulation data. - - Returns dict keyed by group letter ("a", "b", "c") with fields: - final_food, food_rate, final_energy, energy_spent, efficiency - """ - last = df.iloc[-1] - num_ticks = max(int(last["tick"]), 1) - initial_energy = float(df.iloc[0]["group_a_energy"]) # same start - - metrics = {} - for grp in ("a", "b", "c"): - food = float(last[f"group_{grp}_food"]) - energy = float(last[f"group_{grp}_energy"]) - spent = initial_energy - energy - metrics[grp] = { - "final_food": food, - "food_rate": food / num_ticks, - "final_energy": energy, - "energy_spent": spent, - "efficiency": food / spent if spent > 0 else 0.0, - } - return metrics - - -def compute_effect_size(metrics: dict) -> dict: - """Compute pairwise differences between groups. - - Returns dict with keys "a_vs_b", "a_vs_c", "b_vs_c", each containing - food_diff, efficiency_diff, and energy_diff. - """ - pairs = [("a", "b"), ("a", "c"), ("b", "c")] - effects = {} - for g1, g2 in pairs: - m1, m2 = metrics[g1], metrics[g2] - effects[f"{g1}_vs_{g2}"] = { - "food_diff": m1["final_food"] - m2["final_food"], - "efficiency_diff": m1["efficiency"] - m2["efficiency"], - "energy_diff": m1["energy_spent"] - m2["energy_spent"], - } - return effects - - -def generate_report(metrics: dict, effects: dict) -> str: - """Produce a human-readable summary report.""" + +def dedupe_by_ordering(rows: Sequence[InferenceRow]) -> List[InferenceRow]: + """Keep only the first row per ordering to make comparison deterministic.""" + seen: Dict[str, InferenceRow] = {} + for row in rows: + if row.ordering not in seen: + seen[row.ordering] = row + + preferred = [o for o in EXPECTED_ORDERINGS if o in seen] + extras = sorted(o for o in seen if o not in EXPECTED_ORDERINGS) + ordered = preferred + extras + return [seen[o] for o in ordered] + + +def normalize_rule_text(text: str) -> str: + cleaned = re.sub(r"[^a-z0-9\s]", " ", text.lower()) + tokens = [tok for tok in cleaned.split() if tok and tok not in STOP_WORDS] + return " ".join(tokens) + + +def tokenize_rule(text: str) -> List[str]: + norm = normalize_rule_text(text) + if not norm: + return [] + return norm.split() + + +def jaccard_similarity(left_tokens: Iterable[str], right_tokens: Iterable[str]) -> float: + left_set = set(left_tokens) + right_set = set(right_tokens) + if not left_set and not right_set: + return 1.0 + union = left_set | right_set + if not union: + return 1.0 + return len(left_set & right_set) / len(union) + + +def sequence_similarity(left_text: str, right_text: str) -> float: + return SequenceMatcher(a=left_text, b=right_text).ratio() + + +def compute_pairwise_similarity(rows: Sequence[InferenceRow]) -> List[PairwiseSimilarity]: + similarities: List[PairwiseSimilarity] = [] + by_ordering = {r.ordering: r for r in rows} + + for left, right in combinations(by_ordering.keys(), 2): + left_norm = normalize_rule_text(by_ordering[left].inferred_rule) + right_norm = normalize_rule_text(by_ordering[right].inferred_rule) + jac = jaccard_similarity(tokenize_rule(left_norm), tokenize_rule(right_norm)) + seq = sequence_similarity(left_norm, right_norm) + similarities.append( + PairwiseSimilarity( + left=left, + right=right, + jaccard=jac, + sequence=seq, + combined=(jac + seq) / 2.0, + ) + ) + + return similarities + + +def similarity_matrix(rows: Sequence[InferenceRow], pairwise: Sequence[PairwiseSimilarity]) -> Dict[str, Dict[str, float]]: + labels = [r.ordering for r in rows] + matrix: Dict[str, Dict[str, float]] = {a: {b: 0.0 for b in labels} for a in labels} + + for label in labels: + matrix[label][label] = 1.0 + + for result in pairwise: + matrix[result.left][result.right] = result.combined + matrix[result.right][result.left] = result.combined + + return matrix + + +def confidence_by_ordering(rows: Sequence[InferenceRow]) -> Dict[str, Optional[float]]: + return {row.ordering: row.confidence for row in rows} + + +def analyze_dependency(rows: Sequence[InferenceRow], threshold: float = 0.35) -> Dict[str, object]: + pairwise = compute_pairwise_similarity(rows) + matrix = similarity_matrix(rows, pairwise) + + avg_similarity = sum(p.combined for p in pairwise) / len(pairwise) if pairwise else 1.0 + dependency_score = 1.0 - avg_similarity + + per_order_mean = {} + for label, peers in matrix.items(): + vals = [v for peer, v in peers.items() if peer != label] + per_order_mean[label] = sum(vals) / len(vals) if vals else 1.0 + + most_divergent = min(per_order_mean, key=per_order_mean.get) if per_order_mean else None + + summary = { + "orderings": [r.ordering for r in rows], + "pairwise": [ + { + "left": p.left, + "right": p.right, + "jaccard": round(p.jaccard, 4), + "sequence": round(p.sequence, 4), + "combined": round(p.combined, 4), + } + for p in pairwise + ], + "similarity_matrix": { + left: {right: round(value, 4) for right, value in row.items()} + for left, row in matrix.items() + }, + "average_similarity": round(avg_similarity, 4), + "order_dependency_score": round(dependency_score, 4), + "threshold": threshold, + "is_order_sensitive": dependency_score >= threshold, + "most_divergent_ordering": most_divergent, + "confidences": confidence_by_ordering(rows), + } + return summary + + +def generate_report(rows: Sequence[InferenceRow], summary: Dict[str, object]) -> str: + by_order = {r.ordering: r for r in rows} lines = [] - lines.append("=" * 60) - lines.append(" Ordering-Matters Simulation Analysis Report") - lines.append("=" * 60) + lines.append("=" * 70) + lines.append("Ordering Matters: Rule Inference Divergence Report") + lines.append("=" * 70) lines.append("") - # Per-group results - for grp in ("a", "b", "c"): - m = metrics[grp] - lines.append(f"--- {GROUP_LABELS[grp]} ---") - lines.append(f" Food collected : {m['final_food']:.0f}") - lines.append(f" Food rate : {m['food_rate']:.2f} /tick") - lines.append(f" Energy spent : {m['energy_spent']:.1f}") - lines.append(f" Efficiency : {m['efficiency']:.3f} food/energy") + for ordering in summary["orderings"]: + row = by_order[ordering] + conf = row.confidence + conf_txt = "n/a" if conf is None else f"{conf:.2f}" + lines.append(f"[{ordering}] confidence={conf_txt}") + lines.append(f" raw rule: {row.inferred_rule}") + lines.append(f" normalized: {normalize_rule_text(row.inferred_rule)}") lines.append("") - # Pairwise comparisons - lines.append("--- Pairwise Comparison ---") - for key, eff in effects.items(): - g1, g2 = key.split("_vs_") - lines.append(f" {g1.upper()} vs {g2.upper()}:") - lines.append(f" Food difference : {eff['food_diff']:+.0f}") - lines.append(f" Efficiency difference : {eff['efficiency_diff']:+.3f}") - lines.append(f" Energy difference : {eff['energy_diff']:+.1f}") - lines.append("") + lines.append("Pairwise similarities (combined = mean(jaccard, sequence)):") + for pair in summary["pairwise"]: + lines.append( + " " + f"{pair['left']} vs {pair['right']}: " + f"jaccard={pair['jaccard']:.4f}, " + f"sequence={pair['sequence']:.4f}, " + f"combined={pair['combined']:.4f}" + ) - # Verdict - ranked = sorted(metrics.items(), key=lambda kv: kv[1]["final_food"], reverse=True) - winner = ranked[0][0] - lines.append(f"Best performing ordering: {GROUP_LABELS[winner]}") - lines.append("=" * 60) + lines.append("") + lines.append(f"Average similarity: {summary['average_similarity']:.4f}") + lines.append(f"Order dependency score: {summary['order_dependency_score']:.4f}") + lines.append(f"Threshold: {summary['threshold']:.2f}") + lines.append(f"Order-sensitive: {summary['is_order_sensitive']}") + lines.append(f"Most divergent ordering: {summary['most_divergent_ordering']}") + lines.append("=" * 70) return "\n".join(lines) -def plot_comparison(df: pd.DataFrame, output_path: str) -> None: - """Generate a multi-panel comparison plot and save to output_path.""" - fig, axes = plt.subplots(1, 2, figsize=(12, 5)) +def plot_similarity(rows: Sequence[InferenceRow], summary: Dict[str, object], output_path: str) -> None: + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception as exc: # pragma: no cover + raise RuntimeError("matplotlib is required for --plot") from exc + + labels = summary["orderings"] + matrix = summary["similarity_matrix"] + heat_data = [[matrix[left][right] for right in labels] for left in labels] + + confidences = summary["confidences"] + conf_values = [(confidences.get(label) or 0.0) for label in labels] + + fig, axes = plt.subplots(1, 2, figsize=(11, 4.5)) - # Food over time ax = axes[0] - ax.plot(df["tick"], df["group_a_food"], "r-", label="A: sense→move→share") - ax.plot(df["tick"], df["group_b_food"], "b-", label="B: share→sense→move") - ax.plot(df["tick"], df["group_c_food"], "g-", label="C: move→share→sense") - ax.set_xlabel("Tick") - ax.set_ylabel("Cumulative Food Collected") - ax.set_title("Food Collection by Rule Ordering") - ax.legend(fontsize=8) - ax.grid(True, alpha=0.3) - - # Energy over time - ax = axes[1] - ax.plot(df["tick"], df["group_a_energy"], "r-", label="A: sense→move→share") - ax.plot(df["tick"], df["group_b_energy"], "b-", label="B: share→sense→move") - ax.plot(df["tick"], df["group_c_energy"], "g-", label="C: move→share→sense") - ax.set_xlabel("Tick") - ax.set_ylabel("Average Energy") - ax.set_title("Energy Expenditure by Rule Ordering") - ax.legend(fontsize=8) - ax.grid(True, alpha=0.3) + im = ax.imshow(heat_data, vmin=0.0, vmax=1.0, cmap="viridis") + ax.set_xticks(range(len(labels)), labels) + ax.set_yticks(range(len(labels)), labels) + ax.set_title("Rule Similarity Matrix") + + for i in range(len(labels)): + for j in range(len(labels)): + value = heat_data[i][j] + ax.text(j, i, f"{value:.2f}", ha="center", va="center", color="white" if value < 0.5 else "black") + + cbar = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04) + cbar.set_label("Similarity") + + ax2 = axes[1] + bars = ax2.bar(labels, conf_values, color=["#1f77b4", "#ff7f0e", "#2ca02c"]) + ax2.set_title("LLM Confidence by Ordering") + ax2.set_ylabel("Confidence") + ax2.set_ylim(0, max(100, max(conf_values) * 1.15 if conf_values else 100)) + for bar, value in zip(bars, conf_values): + ax2.text(bar.get_x() + bar.get_width() / 2, value + 1, f"{value:.1f}", ha="center", va="bottom", fontsize=9) fig.tight_layout() fig.savefig(output_path, dpi=150) plt.close(fig) -def main(argv=None): - """CLI entry point.""" - parser = argparse.ArgumentParser( - description="Analyze ordering-matters simulation output." - ) - parser.add_argument("csv_file", help="Path to simulation CSV export") - parser.add_argument("--plot", metavar="FILE", help="Save comparison plot to FILE") - args = parser.parse_args(argv) +def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Analyze ordering-sensitivity of inferred rules.") + parser.add_argument("csv_file", help="CSV with ordering/inferred_rule/confidence columns") + parser.add_argument("--threshold", type=float, default=0.35, help="order dependency threshold (default: 0.35)") + parser.add_argument("--json", metavar="FILE", help="write machine-readable summary JSON") + parser.add_argument("--plot", metavar="FILE", help="write similarity/confidence plot PNG") + return parser.parse_args(argv) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parse_args(argv) - df = load_simulation_data(args.csv_file) - metrics = compute_group_metrics(df) - effects = compute_effect_size(metrics) - report = generate_report(metrics, effects) + rows = load_inference_data(args.csv_file) + summary = analyze_dependency(rows, threshold=args.threshold) + report = generate_report(rows, summary) print(report) + if args.json: + with open(args.json, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2) + print(f"JSON summary written to {args.json}") + if args.plot: - plot_comparison(df, args.plot) - print(f"\nPlot saved to {args.plot}") + plot_similarity(rows, summary, args.plot) + print(f"Plot written to {args.plot}") + + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) From 094c3ac32f6b3fc6b6e81c02f6601a3271c87487 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:40 -0600 Subject: [PATCH 6/9] test(ordering-matters): validate ordering sensitivity analysis pipeline --- demos/ordering-matters/tests/test_analysis.py | 253 +++++++----------- 1 file changed, 101 insertions(+), 152 deletions(-) diff --git a/demos/ordering-matters/tests/test_analysis.py b/demos/ordering-matters/tests/test_analysis.py index be2e042..af99ab1 100644 --- a/demos/ordering-matters/tests/test_analysis.py +++ b/demos/ordering-matters/tests/test_analysis.py @@ -1,167 +1,116 @@ -# ABOUTME: Unit tests for ordering-matters analysis script. -# ABOUTME: Validates CSV parsing, metric computation, effect size, and report generation. +# ABOUTME: Unit tests for ordering-matters rule-divergence analysis. +# ABOUTME: Validates loading, normalization, similarity scoring, and outputs. + +from __future__ import annotations import csv +import json import os -import tempfile +import sys +from pathlib import Path import pytest -# Import module under test (parent directory) -import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) import analysis @pytest.fixture -def sample_csv(tmp_path): - """Create a minimal simulation output CSV for testing.""" - filepath = tmp_path / "sim_output.csv" +def inference_csv(tmp_path: Path) -> Path: + path = tmp_path / "inference.csv" + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(["ordering", "inferred_rule", "confidence"]) + writer.writerow(["forward", "Agents seek food, avoid crowding, and maintain heading.", "88"]) + writer.writerow(["reversed", "Agents wander first, then correct toward goals late.", "61"]) + writer.writerow(["shuffled", "Movement appears noisy with weak local goal-seeking.", "54"]) + return path + + +def test_load_inference_data_reads_rows(inference_csv: Path) -> None: + rows = analysis.load_inference_data(str(inference_csv)) + assert [r.ordering for r in rows] == ["forward", "reversed", "shuffled"] + assert rows[0].confidence == pytest.approx(88.0) + + +def test_load_inference_data_missing_columns(tmp_path: Path) -> None: + path = tmp_path / "bad.csv" + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(["ordering", "note"]) + writer.writerow(["forward", "x"]) + + with pytest.raises(ValueError, match="Missing required columns"): + analysis.load_inference_data(str(path)) + + +def test_dedupe_by_ordering_keeps_first() -> None: rows = [ - ["tick", "group_a_food", "group_b_food", "group_c_food", - "group_a_energy", "group_b_energy", "group_c_energy"], - [0, 0, 0, 0, 100.0, 100.0, 100.0], - [1, 2, 0, 1, 98.5, 99.0, 97.0], - [2, 5, 1, 2, 96.0, 98.0, 94.5], - [3, 9, 3, 4, 93.0, 96.5, 92.0], - [4, 14, 5, 7, 90.0, 95.0, 89.5], - [5, 20, 8, 10, 87.0, 93.5, 87.0], + analysis.InferenceRow("forward", "first forward", 50), + analysis.InferenceRow("forward", "second forward", 99), + analysis.InferenceRow("reversed", "rev", 30), ] - with open(filepath, "w", newline="") as f: - writer = csv.writer(f) - writer.writerows(rows) - return str(filepath) + deduped = analysis.dedupe_by_ordering(rows) + assert len(deduped) == 2 + assert deduped[0].inferred_rule == "first forward" -@pytest.fixture -def sample_df(sample_csv): - """Load the sample CSV as a DataFrame.""" - return analysis.load_simulation_data(sample_csv) - - -# ── load_simulation_data ───────────────────────────────────── - -class TestLoadSimulationData: - def test_loads_valid_csv(self, sample_csv): - df = analysis.load_simulation_data(sample_csv) - assert len(df) == 6 - assert "tick" in df.columns - assert "group_a_food" in df.columns - - def test_raises_on_missing_file(self): - with pytest.raises(FileNotFoundError): - analysis.load_simulation_data("/nonexistent/file.csv") - - def test_column_types_are_numeric(self, sample_csv): - df = analysis.load_simulation_data(sample_csv) - assert df["tick"].dtype in ("int64", "float64") - assert df["group_a_food"].dtype in ("int64", "float64") - assert df["group_a_energy"].dtype == "float64" - - def test_raises_on_missing_columns(self, tmp_path): - filepath = tmp_path / "bad.csv" - with open(filepath, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(["tick", "unrelated"]) - writer.writerow([0, 1]) - with pytest.raises(ValueError, match="Missing required columns"): - analysis.load_simulation_data(str(filepath)) - - -# ── compute_group_metrics ──────────────────────────────────── - -class TestComputeGroupMetrics: - def test_returns_all_groups(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - for grp in ("a", "b", "c"): - assert grp in metrics - - def test_final_food_values(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - assert metrics["a"]["final_food"] == 20 - assert metrics["b"]["final_food"] == 8 - assert metrics["c"]["final_food"] == 10 - - def test_food_rate_positive(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - for grp in ("a", "b", "c"): - assert metrics[grp]["food_rate"] > 0 - - def test_energy_efficiency(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - # efficiency = food_collected / energy_spent - assert metrics["a"]["efficiency"] == pytest.approx(20.0 / 13.0, rel=0.01) - - def test_single_tick_data(self, tmp_path): - filepath = tmp_path / "single.csv" - rows = [ - ["tick", "group_a_food", "group_b_food", "group_c_food", - "group_a_energy", "group_b_energy", "group_c_energy"], - [0, 5, 3, 4, 95.0, 97.0, 96.0], - ] - with open(filepath, "w", newline="") as f: - writer = csv.writer(f) - writer.writerows(rows) - df = analysis.load_simulation_data(str(filepath)) - metrics = analysis.compute_group_metrics(df) - assert metrics["a"]["final_food"] == 5 - - -# ── compute_effect_size ────────────────────────────────────── - -class TestComputeEffectSize: - def test_returns_pairwise_comparisons(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - effects = analysis.compute_effect_size(metrics) - assert "a_vs_b" in effects - assert "a_vs_c" in effects - assert "b_vs_c" in effects - - def test_effect_direction(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - effects = analysis.compute_effect_size(metrics) - # Group A collected more than B, so a_vs_b should be positive - assert effects["a_vs_b"]["food_diff"] > 0 - - def test_identical_groups_zero_effect(self): - metrics = { - "a": {"final_food": 10, "food_rate": 2.0, "efficiency": 1.0, - "final_energy": 90.0, "energy_spent": 10.0}, - "b": {"final_food": 10, "food_rate": 2.0, "efficiency": 1.0, - "final_energy": 90.0, "energy_spent": 10.0}, - "c": {"final_food": 10, "food_rate": 2.0, "efficiency": 1.0, - "final_energy": 90.0, "energy_spent": 10.0}, - } - effects = analysis.compute_effect_size(metrics) - assert effects["a_vs_b"]["food_diff"] == 0 - assert effects["a_vs_b"]["efficiency_diff"] == pytest.approx(0.0) - - -# ── generate_report ────────────────────────────────────────── - -class TestGenerateReport: - def test_report_contains_all_sections(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - effects = analysis.compute_effect_size(metrics) - report = analysis.generate_report(metrics, effects) - assert "Group A" in report - assert "Group B" in report - assert "Group C" in report - assert "Effect" in report or "Comparison" in report - - def test_report_is_string(self, sample_df): - metrics = analysis.compute_group_metrics(sample_df) - effects = analysis.compute_effect_size(metrics) - report = analysis.generate_report(metrics, effects) - assert isinstance(report, str) - assert len(report) > 100 - - -# ── plot_comparison (smoke test) ───────────────────────────── - -class TestPlotComparison: - def test_creates_output_file(self, sample_df, tmp_path): - outpath = str(tmp_path / "plot.png") - analysis.plot_comparison(sample_df, outpath) - assert os.path.exists(outpath) - assert os.path.getsize(outpath) > 0 +def test_normalize_rule_text_removes_noise() -> None: + value = analysis.normalize_rule_text("The agents, in a line, move TO food!") + assert value == "agents line move food" + + +def test_similarity_scores_are_bounded(inference_csv: Path) -> None: + rows = analysis.load_inference_data(str(inference_csv)) + pairwise = analysis.compute_pairwise_similarity(rows) + assert len(pairwise) == 3 + for pair in pairwise: + assert 0.0 <= pair.jaccard <= 1.0 + assert 0.0 <= pair.sequence <= 1.0 + assert 0.0 <= pair.combined <= 1.0 + + +def test_analyze_dependency_detects_order_sensitivity(inference_csv: Path) -> None: + rows = analysis.load_inference_data(str(inference_csv)) + summary = analysis.analyze_dependency(rows, threshold=0.2) + assert summary["is_order_sensitive"] is True + assert summary["order_dependency_score"] > 0 + + +def test_analyze_dependency_not_sensitive_when_same_rule() -> None: + rows = [ + analysis.InferenceRow("forward", "seek food avoid crowding", 70), + analysis.InferenceRow("reversed", "seek food avoid crowding", 72), + analysis.InferenceRow("shuffled", "seek food avoid crowding", 68), + ] + summary = analysis.analyze_dependency(rows, threshold=0.2) + assert summary["is_order_sensitive"] is False + assert summary["order_dependency_score"] == pytest.approx(0.0) + + +def test_generate_report_contains_key_fields(inference_csv: Path) -> None: + rows = analysis.load_inference_data(str(inference_csv)) + summary = analysis.analyze_dependency(rows) + report = analysis.generate_report(rows, summary) + assert "Ordering Matters" in report + assert "Order dependency score" in report + assert "forward" in report + + +def test_main_writes_json(inference_csv: Path, tmp_path: Path) -> None: + out_json = tmp_path / "summary.json" + exit_code = analysis.main([str(inference_csv), "--json", str(out_json)]) + assert exit_code == 0 + payload = json.loads(out_json.read_text(encoding="utf-8")) + assert "order_dependency_score" in payload + assert payload["orderings"] == ["forward", "reversed", "shuffled"] + + +def test_plot_similarity_smoke(inference_csv: Path, tmp_path: Path) -> None: + pytest.importorskip("matplotlib") + rows = analysis.load_inference_data(str(inference_csv)) + summary = analysis.analyze_dependency(rows) + plot_path = tmp_path / "plot.png" + analysis.plot_similarity(rows, summary, str(plot_path)) + assert plot_path.exists() + assert plot_path.stat().st_size > 0 From 9c79968e5095861424c788fbf4ffe94ed5dad452 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:46 -0600 Subject: [PATCH 7/9] data(results): add sample trajectory and inference datasets --- demos/ordering-matters/results/sample-inference.csv | 4 ++++ demos/ordering-matters/results/sample-trajectories.csv | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 demos/ordering-matters/results/sample-inference.csv create mode 100644 demos/ordering-matters/results/sample-trajectories.csv diff --git a/demos/ordering-matters/results/sample-inference.csv b/demos/ordering-matters/results/sample-inference.csv new file mode 100644 index 0000000..77f06e8 --- /dev/null +++ b/demos/ordering-matters/results/sample-inference.csv @@ -0,0 +1,4 @@ +ordering,inferred_rule,confidence +forward,"Agents prioritize nearby food seeking, deviate when local crowding is high, then stabilize heading with mild stochastic drift.",87 +reversed,"Agents appear to wander first and only later reorient toward goals, suggesting delayed correction and weaker immediate goal pursuit.",63 +shuffled,"Policy seems mostly noisy with intermittent short-range attraction to food and occasional anti-crowding turns.",55 diff --git a/demos/ordering-matters/results/sample-trajectories.csv b/demos/ordering-matters/results/sample-trajectories.csv new file mode 100644 index 0000000..76de75b --- /dev/null +++ b/demos/ordering-matters/results/sample-trajectories.csv @@ -0,0 +1,10 @@ +ordering,tick,agent_id,x,y,heading,energy,food_collected +forward,0,0,-3.1,1.2,44.0,99.2,0 +forward,1,0,-2.2,1.9,41.6,98.3,0 +forward,2,0,-1.0,2.4,37.8,97.5,1 +reversed,2,0,-1.0,2.4,37.8,97.5,1 +reversed,1,0,-2.2,1.9,41.6,98.3,0 +reversed,0,0,-3.1,1.2,44.0,99.2,0 +shuffled,1,0,-2.2,1.9,41.6,98.3,0 +shuffled,0,0,-3.1,1.2,44.0,99.2,0 +shuffled,2,0,-1.0,2.4,37.8,97.5,1 From 2294ab875141796f0d92546149c2e86a4562df55 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 26 Feb 2026 21:26:50 -0600 Subject: [PATCH 8/9] data(results): include sample analysis summary and visualization --- .../results/analysis-summary.json | 57 ++++++++++++++++++ .../results/rule-similarity.png | Bin 0 -> 52389 bytes 2 files changed, 57 insertions(+) create mode 100644 demos/ordering-matters/results/analysis-summary.json create mode 100644 demos/ordering-matters/results/rule-similarity.png diff --git a/demos/ordering-matters/results/analysis-summary.json b/demos/ordering-matters/results/analysis-summary.json new file mode 100644 index 0000000..f5d6d01 --- /dev/null +++ b/demos/ordering-matters/results/analysis-summary.json @@ -0,0 +1,57 @@ +{ + "orderings": [ + "forward", + "reversed", + "shuffled" + ], + "pairwise": [ + { + "left": "forward", + "right": "reversed", + "jaccard": 0.0333, + "sequence": 0.3406, + "combined": 0.187 + }, + { + "left": "forward", + "right": "shuffled", + "jaccard": 0.0769, + "sequence": 0.2899, + "combined": 0.1834 + }, + { + "left": "reversed", + "right": "shuffled", + "jaccard": 0.0, + "sequence": 0.3426, + "combined": 0.1713 + } + ], + "similarity_matrix": { + "forward": { + "forward": 1.0, + "reversed": 0.187, + "shuffled": 0.1834 + }, + "reversed": { + "forward": 0.187, + "reversed": 1.0, + "shuffled": 0.1713 + }, + "shuffled": { + "forward": 0.1834, + "reversed": 0.1713, + "shuffled": 1.0 + } + }, + "average_similarity": 0.1806, + "order_dependency_score": 0.8194, + "threshold": 0.35, + "is_order_sensitive": true, + "most_divergent_ordering": "shuffled", + "confidences": { + "forward": 87.0, + "reversed": 63.0, + "shuffled": 55.0 + } +} \ No newline at end of file diff --git a/demos/ordering-matters/results/rule-similarity.png b/demos/ordering-matters/results/rule-similarity.png new file mode 100644 index 0000000000000000000000000000000000000000..6df7d65ede75c998decf1a3c32ea11fcb4b397b4 GIT binary patch literal 52389 zcmd>mg;!K<_cnF{3Kor|NJ)1n(j}mDNH<955Eh^`NDE4el+uksHw-{SM_%9YnU$@hmAL_>v$4H{xvdQc3)?*wUM5O2N5^Ll z{H(0j|9S_Dt-UGhcP*!OILd`*QW_38I5!Ef|4x`Db;G$@uS!3>|JXHQapa8WlWCG~ ztETr*L{ZrK=)$iP)it)bo@arA%$B(41ikIJiB$KKnN}TEuo-(+6v|hQc zebc3CWWY;LWY(bsxt;J+aK){Cy;7GuzKssf`RDOWU|zfG)Zc&MJPQ=X`}+@{c&gBU zy)vGDe(LWRa7r0Y^8M=%n>YWzd_?pj5xbGUtDBp2ibRm$!mT{rs-eXayxd2rQdbL> znj+}w%``Q6xv#9RNQcxMVot&(Qf^L_j!1Q08VaYCz2QMJj#L!kLb`7)ntSzk)JiR^ znFnMPXX!MN4drN0imp+Io}%m>`z=o9i9pcmReN^l8_Vu)p=_s@M=HXJ$poC`Mm^W% z%Z?5)v|7bh;kU)kUbuKM{LOVnnGA*Za@0l*!DP9kZI35AM2rrYwltcFY3 z+Kro|St-n8r_vi9`lU#Q&|%6?;NS=rAtn~59d}k6C?>Vj<>R@-qNEpuhIiLzr&B_N z!)My!WqnVcsT~{BV>f9v)9thvtFH1q*zC98T{E`hp_UAe)U9%l#mvX^*vc#qm&t3% zY?D2kYJSH!Q*Pdu9Xr*LG$!cMm?%_X?!ArfS1h(h)%%_br5TmhBE=DWG3uz5t5ZQ| z+MdAn69&vQ*$~2$M7$`)Tzz!t;kGqsI$a+`vSqhN7kgN3+0Vk!8p|Ph@g@stj@^wb zS18kD-VvxVsAk9P?CwfGefps^ejqsLtoOyv$IA=v z{}v`o;Qc954h{Gg?=e(h1Yu42H zFT5>1nJR6oE2xsZESEDlINzw}F5Pn)0Q;ZXgQCmh2E@mYj&4qZF5v{&aT_{=VvBZShPziXWQR%h*-w6SzM7c z50zPw-Xt`SH~HnbG}L?AP9~a#S>#|N!rU9Z>4n*{jtD@eNaAB^zQPPQ3`xi`A72NE zDcv-P&K&c7`8ZCt8Z92JOy8)jik5YNF9fpN*w+^G1;0$%|?+7h~f|Tu$5{WPGl_pFS6uMUt?;r~hi3Gmp}J zaTYiH?vu|k$_ARs*=AkIYdtE8wB=LpqIU9SH0qutwkpZESdmdn98{zGayV?e%sZuQ zRX084V(#7UW`~8t&GnVbCpE39u!FqqT0lUqR>@T@YGfGfyT%3zZ>x6mrf}M}L$jY* zR)hKJC*1aTR&Pc38m6Io_$P);!>gY>sd9Hh4)mdM)3>#j?JR({6R>>Y|*CLfp-st4Qfs+;Dl*V>w+-e@;$jsy6ieXD6 zgJJX0Xt@*PP;v5T#sVTg1=GJ%c%3WeNr{b)gJRwidz)Oh-)`L3YZTrGo6!u>I>`?& zAcMP$gH*LU<>-aln|4Q1_F%+DbDbYel8TaU>ZsOuzrDfSDpXWV9d4OKD5LkoYT(Pq zL00SaCt0fU^}%E$OYYAkFz8-2J+2eVLe)d{y(^><8A)D4y26k0^n_He(x%F%a#G7> z%r-?d9FqR3yQx=b8c`=+ld+}uq{7+SfoXG04fc=xt~hQrfAUhc|A8r)u;)O6?SV^j zOT+eGE%#j*3Qdz4h*@0N}&5lCu z?#Ru`%BlrX7ePce>N<(baU`%(Y3S9zo7}z*mxFJ(I#(+LT};3(AL3AJ7)>pSSD<5= zVBgZ}GPoF4bpL%T0(?g-Jp|@THa?E3%B%h*s!F13>#mhN5TqnlLJct}^{< zP}jO**_R!_Z3|$aweX{upF}y2gChk|yrS@KZMBu$_ULMFjkhUXL(tChATql=>Y6)krm5~3hUnd;db>}}2nb3t0%a@#PK+|K)2bbV7@WOphe zY)fb5RZejeHc^>ssh~2iqS)Q>bFE3y+m&wXL&dgZiB@Am$Icyi7{R?28C3Aqyg_{7oo~Mm zwbnh44z^7y`pWuDiLNHSt#uC4vM#^>;_Tp1k)>%(dp36wS8b0Cve*V9Bh^w(uXFzx zEU30ys30HuW9ain@kN7sOwQ3yd^|j7O-fV9Dcs(0@!BR5g*+B{x&hx5RQ>*OX)8YV z(~0!bjm1IaVU=XyE&U1v#yS6-eRjM(Lf!jt-;HA_$@}mt1xFQMFNs^#M|T*Pbu_CU z_j-abVh9_uIYLI~G$tdcLy>uUBB+~sp85jM#(kJMIeP+;e!CEX@@+W)fX(=NKkk!H z9Q9xLfG(Lqq94>~d4>d~Sbn!(o09gJT<_QOcH#}uCF$cXwi&UIwW5?1+U!xmv(?=^ ztDWaorBv;8;GW={0LW*^q%}rgz+om3kwBir=P)xM^X{&36uR@{i=W0ho0{M6Nm7xk z2_5EWl@u?HRP+n$bY#*Nli1_-sUPKzx}%x>?kAQzq3Z|CAZbhG_>yiysN}O6n-f#2 zI*0Vw0q{i65SoIIbSl#>vmXqfSnm)iS@~JeA#7QL--{ph=`+-uk-B<#2!UqOZ!$Ik!W+@1Tm2Jw?`MKZF;MXVLpP))+v-`%m0m0fh96 zCqf`H#ckzwbALJOx^YX=Q=jv$bp)&ilpK2CYszkWA75Ye`luJi8dH<^=|qM95w{}+ z7tW1Ux4Odv4B@rLuJN~t{Q$^OsIk3+!UJ<5HUEJdg@W6|4q=G`>iv1!a$_H`JD&=;C=yU@%sV2pdiBici^E zLdE;M86AKq$j~=ENtoUFYd#|>m|MrTXQg6A7$aET_G!Ze)hxKTT(J_)97;$s!dlf7 zWFCLD!0iT;`jtudaNDmC4KL1?TPYuY%Q7yg$QlM{B)}iBqE&rx$WP; znC>Bg@X0~Ob6G}Z`z8RGAKI=0jEWcCyKts&FNGzKNTTo59Haw2ZCyRpg6AiQdt};I zbn|qDJkGvNiC!j^UJ!1eop5s@ds_s zlYDGv1~>~K{W@;Q-A6g7sz#h$H9&FulBRNRgq@<4?=1h?_hD0|C<#C3=VvU^1oWSV z+!D8JI)prvo)qkR>RdfJ`UXsDtDX@Z;j{a*r;3ob_!oA zn3PYhN}VkFnr-E(M4=r7YaXtwcYX7{)#l0FICD8QHiS^vrq9>xK6rJJ){O#Ns)cQN zcV%mElzAPL&zsg7Y-YxgZoeKXdDDG(m%y^_&2{_L3GsCYxdW|Ah>zhswz})<8}mrA z_7sG3ZO^Dhf3D7(LP@W~T~kbT&3d~4TX9NZ&NUU0C~QzOZ5Rq-e^o(leeg`dE{UO=59;ZK^5a z>!?(EA$BVnxGcA%bb^Gbv~G)jqV7dN!%gkmN2y3z_RJ{iB! z^2bemM+m$+l`g-qDFt6}URFAIPHdS@KJH=MSeL9On~+(1G;7js;QW1e`$dn{i8`3% z6X6g->xNW^D_m>?mmZS6#v`+sL1Ol%_Llo(HQcQ>&wSgF$wdUqS~Wb@yHOZy-h=)= zxRf__MJ0Z&F^qa^-9+aoxR%ozRcuWclcHH@c6G9>8{I&F+?5G-)@qC6GEBIEbSj4| zyRK2?zIiK&XKuF-eMZWyg=t@o4$nR9YQ2njfh8E&ZK>rG1qX+JFQ7Otgh5Wcl`)S}U41``WjnMt zf~XkP4Y@=M?=MVF8hYVhxWIOwjm%(TE+~B}-8aNE3DLKhXX0PE*37tN9w(Gscr~o2 zB%bOlBNvHfdjg;R=E4&UNwg}puyFtA3w&X-jyWZFIj)#!AWblBMTso~f2N~z{Sa{Z zKta}<8_ai;5L+7I{g-fX#^*PTS*p~!{Z`_9KQ5g}8zf*e4xo8ya)fRXlD-g{AmTmh zS-+XD1M~=08^sgPA?mpAVF?7~D{KJP^eZ;M39&8J)_S^F(`yk?vk6fuHo>*mZs>_4 z-*lJC+=|vk_q7BJ6^GcWD%56wh+*$HupkuQxA{sIQku*9uE+FhQ$n0 z-%6{W#AGNGmvYSRIf<3Ff9%hwN%=r;YwMD&OJ@jX^8TTj11Sk@X?r& zQ^ymsT)y*?CWnG|u?&%!7bXag(KDAZc(VZ>Q8OUGicn3D48$0KX=HrawKyQaFv*1K|wYF0btsf*H@fH@D*3FZyR9O0i6TzE<@3Fl&Xt;tinoN7{vDJJjfLWQ} z+&P;0#Jrm2j+v!z-tp(!(z)@YNzKPZgU!CMq!r|5Ki=ChNmGAsW1(Y$`%>LjjLwuh zSR7N_)#HuOV){rnSnp)D8B{)yuao^j(cTDM>t|wUKKr4sW`$NZN(yKx`L^X4{-37- zo62&UrA((0I@sPyq6~9>ko)>F5y6UwD6{-^yQ6H;Hb};#?RnM=SsN>m|5%rQ47pVK z3488*`RP_T{B`RIftHq11^{GKW4!b{}Rwa?v zqYG4LXQo@JIjTMwds!rM!EOw(J=pS9oC!hpM9y08pv2%ExnAkBkFLp|pJWYT9=u@~ z-`O7XR#C;xBmm539_Rc#gJFE3YJ2qUWA7s6G?^KPmMCUC+mjpLzibJFKcbb1{E|U` zwBmjAmLrS)<7V1sI&*Hd>BdqdpjwIYt7}F>`k#ir8v6a5*Ho6Xoly>V{*qZ((7MwZ zcn5(|;;7yr?jbpfTrg6x?JEko#hjIqcDkq|eu}?%sQm*M9IwIjrJ z7VQCDjHbvWrkE6QT^m@JK0>d7VBvXO!rmh3pv|4!Gin0f6ma-Sa+D~v3w;H!1bb? z6>^4@kp@-S%xuZ1d&}vBiJ!3;QY0H0pM5G{y5^Ig&6Rc%7|WgCzh+(u>qO^rq_2Ob zlZ&Awq*ve+ft7_ktpZ#~IJ!ISZB-EZj%zAwrp1@$t4o;1z3M%l@txJluQsd<2%WUZ zU^0O?!`%A`F?;uW*TzLH9%A{jZD2FvNT}MFI^!ANI!EU6IJ=8|=8;vsz|d0*3^31^IKimgVNVsG(k%!E9ym zCUMm5z83qThGY{fGm)UHr%nys&?qzubGxHe<8>glZ92r0a{b%xFF%S@mzB}2;K5BV zdmz5jr}Da&Xk0WVQacg-_VqH1!gF30W8pG2HN{T~5y{(G!E>(6Zl>>eZIJ0fnJ4v) zndUR9AARz@v=ky0qHo`QZqm!k%l`NHeTPwiLMGnQk2st*XeEUx`ec3EhfNXRed4pRw>Z8HY0w#P_k|95Pu zM1s<8UXZX&94>P{`xMI{bIJ@NdckgqwQ(I^a)WG07>gcIC+oxC-MOE>6Yq&0<9D1J zSG}+Nw)&H4(Nf{b7l*i)AfMNuYq)UfEWRGe&MuQj^_1AyLe7> z<-jqX>M^a&xvrnBDM?E6Hhe#Ir`yi6b>BZw4Mz`qx=7C50*wUpHxxQY!|{GD{k5P9&1 zq36$^FCwd|s@gV<4i3Mcqo{S?2SQr_QwciLmuZksNF%UCKc-aT8nW1=Bhgx~0aTC_ zW|xCKC$0JJv^K{oRRBxTKGYC-Lo1geN?!x5K#Q{fI#MA(usXU3*e+l%S6^Sh*kugC zqltp*(8-^d19$sMp;;z%#hT$8xm>X_;4bW8VEsa3&t0$%2teH;z6hS{FotM>XDAdQQz83n`Jz&ok2EwsK7KN$EYbhELZ8k<<{zedIRUj zBW|XzI+LsB-VM&ou=Ltgp*Y1<&pCZ)K)E}oSRY{}f+P_LhU(jdziEu7aK6>hwrBIV z#--bK@d({u)wSZM>LE~y3wo4qB)vPA5@H0S3}|mu+It;gGBrm+6Za$6HgJx_F;4tZ zx*U z3S4r*fdq;JoiVA;N-3{BH+$8#f>%`H(!W@$`Y*pAw_Ord_KO(}?6LeT{`Q9Sf)SF) zR&ZWRchB~_dxiX1H*(Disxltimvs_(hKLr~f^Qpb_j2NM8fbFcEeRitjeG9RXS8*c zCqc0l%pGE*H8$5H{1IEvum@tr5ca|e-)iQ(=^(CCq9_%2*35%7vv`>mHdKkN?dLOQ z#xHF~%Eh%Bu$orXi>T&q?x1R?yx8+WBs|=GjbIDjJDR;m$hvkEY$WU?TR$^2r0Ji)ln(oEh))}>QX+wHg*$aw7O(dE_ zF;Q?P#+>9Kl@@H9cEJ>BDI?3Xu$rs>HMtDWag2o9%Ive0*-U)z zS$d9hnE}xf4%$_OP?d?7*p6}IQpum_kDse`F1!F&66eFg8Mga+%sIIySeR2XSW8c4 zo*)7-ywu;s-|ULy*kxH6`grX7aEA2Gcg!Fu6}A=!rw=hJv!Rr|l%S{kTp|UIfAs3- zsM}KieyjQSBFla`4L;_Kn_mOVqJDe90x@n3y*;xXt>?wpZtvkk|FMu52S*`wGa-bO zj|$|%cVE8TV|etz2LS!sp(^YdMKPD`I>+Uqm|v{XLKvI-G$bP|&AnwtM@LB&Q;A{O z{kH(jZRlr*B;%++!W7~ zS7?n=HEB;UKl_|l%DS-;2nn1}17J1j7*(@F+v0iRe}SxMQrTuA+q1`_Q*I<17OI!Cg(Z&N`^z-c| zB~)gsfm`SXieP@UN`M|%PaJbRfNk(#NJn6{*n#FXn1eNLP*s`bCxbU1=Nsfz?92*o&}Z+3yHnudYe1E(DSQ7Vke9?JEH zXY2u~-5v8js&ZT$P{MM(tg)-zvaC&jV_1aTH|`BmJPUCs1DVx)w6bifG3@8*L7B)q z`t|6JAY!g?5G#&{Z<>_*rAQS3e0P!L{i4S+eTtiuh90nl8_{WFw zj95gZ22Km5F7=s<*Y16f2MSmgBostkZr;=H&*{#8I0&6?e=l%mi}Zp}H1Nt%px-vn zb*0FgC_Ds>A%e}YK4y`)E&r?MS`+;mR+1^-))Zz<1|}=+TRW7@ccA*5dI{&GY?GQ@ zAcJ3Ud`g+Lid!RxK4WtsW$@bOCf9q^>({TF!QQ|dV>1pk$r=cTaY+I$%cR!jTr3u% zCIC0tSU|TTAr$PEqhRjBnrBk4p9Uul?3yYNVPiUA74j$81dWwBWT6z-eyGG~lF#hcXo)CPp`|=+P%5Y+oic0YFlUExk1e{wlF|TUh z=YfzFS8WIDkOs;Ij>$s31dwxT%SmU6;DhE$t}7pW-eN(5V}d650Mu5%;3N8+Torw4 zzQD92eMiY-?xWw-?_WPB7FTW}4+bH_zAtCJef#zd2PNgVUyeff%@;3zE1KZf>*K5N z+|B1R1%y=08j&!I<=)v$F7IM6NUQ|yBUy-hVd@*R*_GMl_h_nZyfSw+pKC6QMykQ= zIH?3YHlvP1#hm^QbDg>jx->6Tie2XBbXM2a?3#;lObZ6B?6;S7@C4yN;jc!u7C4q8 zA5fmN`Cd8iwi3FiDJFrZTr(3QIKQBSlkn4IQi{44RcIY}?QSH}9EGH8&*ggcK0oIo zZb*RZO$YZGc{#|mM#8qkrN5q)_nLLL;*}ol+B>#q8)r}F18G0YZWl1WSH$i#JiP}q zcrvjBoETn)*5sv|&I%tcNek!^hj!CWgb zoHOQpMtJMiEwp`tDSRqb{O?HyP!ij*genn$Nrq&SjGbiQKxM$@G7Tbx{X$QsYf|1o zdqGqdR&TED=um|1+w>83oM;@#>~OH(SK_D%EiO4UkL&L8QsrnCg*(o7Hvu}D+TR7e zYNQ&Wb%RCw4sf>7ko798C{`&zIWq4~4cK)9n@1O38&nppra*8}2id}EOr5{VtXU`p zX^$3CJ;!(K*QCtB259)Y)`4UKs#q%7wtCkjbrl>P?;6NGYpJ|JL2qFWKs1i&u zYaoJ1T)Oyeb5{9G%59snz7G_#y$9l zSFa3N%72Prx1YO>eKXGRDJ)$0>+#>-xnyjH#~Hu>bQ*INURx1!j~#NE5TO^gzXjpEz|^3LY1| zRA3frQ;ydeOdmfuK|$d80Gmr0>?BM;?X%yH0!pv(5^LqtdSLWXt*B~*2Llkwg-!`V z9;)>qOg(2mbLmE4zjFm39uAY%P{44~z+)HzdCYFnb7%02{)=qgDosyM&+vl^FhSCs z`D(eR*$x#{27e~K<*^9h)$ z-eN6SN@IXIDQT}BuRH+WYEbvBmqBogco0``Pqs0^>VE#QZb-2oA=MS~#KM zg`ROH-E!~wpL|x$xr;5m5K~OLmFl2*-R3ZD=hR^TYCQP~fOEa22cQxAI5W6w}X^r?I z`%g~_k?}kF_ZMf&#m77s>)QdeL^FXcVp2(EvapN%@5Pw9Ggi>~Xnv?D3m{hghEB@C zFYVH2_hC`6PSyuSK*Edj38DtqR5%vL{=^qyI&tDe0u#*8=dK5T0+GTtWU^+M%V|KE zq2xmDSk;vV+%8ymTocf5SQBgdqNbH`uVdLxKD&*AAtGWhZPmM+2F`2_a4Vpuo5xjHCM5E&Zi?irRLFvVDq z&1Ql&)8Jxd%~Lg7_?bQxdzfq79>--#2g+SgVJrA)vC|wX5kxEvMgjxv5|d$0 zo8f_tj~rm`#Owevl8t83nF5nqID9J3dU2#e4NM7?MHao`pz3DKxj+b%y8KrZ^+Cc( z{)oxbn`;2i-*#e$#J_Y3qpg3i&BxIj%W0yPVSQKD_el7^6xK>xKVkew+I{ z9ap}^){8j*BY#@y5?K7NeDK#1KK@@vc=2-3^(KgfP=pZt<%8!{3gfxqQ2C23*sG@= zn~X%`B>$|5E8MI0niPD0M|!2`wtQ|Bh) zn9*PCA3t*M?xPsl=Nq!@p-^eMHclI~aJDhjcQ8#(I*A4$LP&%jJYQjM|7PO*^!s=*bV*`=;$@nwfMe zdfyQjy^$`qB7#sEpKslAHJ0D)JtUr25wE6d`r@MwRy4YWXH@ zx4d#h*2J@0uHOmOAxCRUetyhMHRxaz4vVO1)$ph>kI|)Nzf5a-wCan|PLvc2_iFgP z)qev5%5%RdS3;_3E=B1nMme21BE0K_N{kefT$btjG1{H`gl@3lC^&F;>kX5ZlUN@s z6XjrmINr?5SA(f79Hz#j-RN+xd+!Tvh$)Y>FTWdn`@pEF8NJrBxUZb@MlXMnqPzIA z@VxO!n%)nZqeAR+N-}AR7Oq9# z^iT{wDmqktds+`|+%g-jC?U*pcqnYYICn;)#-_%ob&B!&-5cRIb*)&O7T!gw>Fw(N zcrd?(2olaT|9ypJ~G>aNDV1Qzw`~!y&+wxw|M= z4fP(R8fZ{t>JBO{=PfddTaregN(i@~Xmj09AQ0zhjC0$iy`#31bOm?H%qK%D$symv`ZlKAr*>f|C-K6> z;vqVXO2<0(tS^EXa?C2ViNX4+0V-=CKRe3AsY``qpMt`xsQi{558({uh4Mb(2- zBmMSLs2fDIip#YP`W*=s^djw-u2N8hu?ja|XZXF=gyN%B3*J~eShTbk-qB5+7AB)O zRNaw$VL$MK_cQWEY^rrjAl(b3WK+u|c2+RV6+Wgprd?#@!U0VSOFFn|Dy>7gBi$G9JB=khzJ8O^Mm z>OXI43eI)qa!2#k&CPdMlcLjt$kKv{aX~g-yeR?yRDf7Iq_f7yI=M*@AgtfQ`n`WLlUVb zOY4osG(fx8-Db+@(vLos_&J8+`-$9D@pk<_d})7WlL%Plx@#ybuExRf*P0L*l&a%= z`tjfwkd<-(yXmnU1K{ykbK+WFFs5*tb&4JyVr&^6e+s=&k43Pmx!MeXBcqj4B&4yD zSdQ8$2L2VGmD2#{D1S8&C*;h0k`;AfLH#~e3+UQY-D$Ejv!9TTWY8|t{Yi-$QwmB- z%Vda%O`k@wl}09*Ixp}Z0PWwMp-4gasS99nc{SFvblL|9=-U~JNl}jTKPEjj6_(*c zQJ0C>r2s+G-n;h*i(OCZVA-+P)Oe9rK4tMw_Dn?UkUb$>R%9Xd3hzA7LV)q7n2^K zB#A5gY{#6MigFbaOYg95$KGARLo?CJPQUvhZXdgYwDsY_CKERVNvz3Qm4PIX?G|PY5m5-|4{XKo7u;GP}vwr%#`L z7M=5Opj>mG!ENP5JG(`#@?gzM;8a`mlfB9T z?flbwr~=9pifUyR`b2BG^kvTuKVP2XY12l#2Z?m1=L?P&1wYPIeYhtmAf2ndo7cg4 zyvD}0BUNQ7amSf73mc8*JYjckQ3ihobJ51~2+FA0bG7a!Ui1;v;3;S9sLmQunT*bL zXcEg8cofpaoAfO0K$v?(ZP4 zRqEUB&u6^w@jip(Pbdut3(0BZ(*u7UZP<=F?o;mV1^v#BYmMy|*P4+y8FhkSvajv8 zSJeJKX^rLh?9D@-<@%sVbclIv!&~Eqqy}1V-$nK{B)w5rzVBLFZARL|Cc(AMQa!^Z zv2#+tiD%iZGV}=uY>1Pt<$CW3EE}sl5woGFJBIatL@205Ockih9pj7C?Ud!*bCp! zpI~Kp%$pHAJJ59`lY1h-Zh8OudhGg`CI4q_2u7X4R>}|BhU#c#pZZv#+#f!Fk+O{} zaNg9NAK3K*HsBFlkmCzci|?Coy1Ux&ajmlDRgK9aiua!YPg%=k@hJmsB6TpOelTek zz^!8n=7l9>;pzCZ?o?4vRN{(K?th^0+PPhSP;!!A$y`t5D?)F?OKyzkwr>LHp#r3N z8hrLEk1G3Lq-LtK?DOv%_FMZQOO6Dp9}FJl_RAcxtoK(7*l*{`L)s$UK~*C)Uwes# za(Yt;+X~A@p10@KDKMaFj}GUy<+pn6UpN)ny?BO_;ZWF7xSMNf(Q@SCHJzH}ouf1T zVm}&8n-`<0P?8g7x>hl5e1mz{LjTHAxT!`rv@+R5YGNo}?qD#oz;npm2-=DuEW#%} zDBNqZd#oyNP8WLCu260WBBlJ1F!69#An8&xeX%f^0g5M8IEVP0RJkJ+ntLR2{d*XP zmCaajQ*x56)ZPYFPWo#_7Y)KPk!pQ3qG?YY~UaaNnOi!#u@ z2VX&(DoPD`8@iK}i!7daP&O>k(_4f#VA~+qGryBEAEXnBs(SMcl_(lafgIYmLV|QH z^<$;ejmqB)ZqoWm746eZGQ}KH+(F~A?7!hQ2^zw-p3lnW_VeiiQB}tw5!1q?n=+_!rdB6H{ckXpxQku3Z4}EsqW$yKLS=N%mZNg-JmrQ+f zS4-~J@25-ztB2>O4QlxL<8F zvVcf<%tL6g*Cf4rT*Z!)-^qy*9Wo7qXdkOx?WvJ(g6sXoI+oiyZWkC5ujh)d&zg*a z9aqC5UIeZ#Q7qSlwcHJ>R}am3ze*Q+3oeH*R06Y z#F~8vHn~2lQU<`C>`qppt{&^-=cIX-D#lVRil~&UAKZ^!L)Z=$t-m~d=8Vjs{qA&k z8k&7#vMy`}|MKO~2d^BYf`wv}bF!S`Wwz|iMn`|Guo`uQNedt4`ymq9Qlgu_znERD z;W1ZLAV*Y2EsnX=Z;*wvrQ|*Sr2Yuz$9S-tjNRJ8)!SS&Ja048J!kBuSB|2$@Lthj zSMr_y@|yV!hK9sBT5YY8OeTE6xpB>-&!$tB2V>tl>%1Qr4Xf2ijYVrVPL3?dq|g~p zcYZGb0<$KPSxwZ_v&tBX$9$_nJcU1bi{&9Dy)V}24Bn-%y0}wHp*MIh^g-LRZZ7roHTCC8l<&+>UxEZin(eX$>V0!$`fte9nufsfF!(Y zOqRd6>5kahnb7>SIf`>llgsZPVavfa*M9!)RCA3?t-Mt23g?#7N_6t-@gJ;W63X&p z{3fpM?zl=4-wb85KQi{V61iLVb~cJ2ZB1sLeRry_O&}kWr;`8dyR$~kvM9T!80C+m zQCrQOqS5SO@BTf4s=j`jMa_=)e9M+XGUYRfFyf=9?_q z_h>sFh=~~k(gsg^-b3L|U0oC1tPM>oA`x_d9*>A+5-;$!C5q_5f(+O@b^H4O9h<2` z1PROsK|zxNCc9Pmah7Tu`M@A_kAorSkp%R{ofK6y=jJkMyy6D_N#pKRX_JHC%Dsgw z6AfJT`Mp!8PPN%V$-w3N9y&vtL1ajN=Pv6HZPo$90yGtU*JL*MaTL4u!{~LYF8MfI zS(GIHVvf4SpiPsp2*!-MlC*8v?_&Ep-XIHX^9ycki&q(lm=zi7}(9O=e!s6 z&xmoVgDv;EA2Ko}Faa!FjnIK+@|<~yzCBs|t=#L5#lZr?cejm*KtzBxLZ0-5_+D>C z5Fq8xDuX0SSfzV>?cu*G0_^tVHyRu7W7U{$pf^L$mQqpAPM^83@|KNF7-2?6#3xk* z6x+nHm@^^qZ&6K@czWrN_u-1SKKEy8Y?&*n`GD~X_UD^&;DRfmod#P#t*);}fJZ?Z zQ0f%8VJ_(QgNY^;OmX3iYAnEPzkrDq!-9Xnw(?;J$+b>li3-PggZ8S#640h)z>|TU z9Rr6)Z#XzPA4`OgGhe^^_zI}l4%JS(RUX^+GH}k&nLo6k>v_a5*+Oby!dyrn;VeKb z_$c8Z;9n1s|oHhA(tdPlAzL38$c+3B?AiOeA_W$!f#y;f(|os*z!Vf zwU*GN>UjSC@)Q7JKa+aDb$>N=xpB;5TV#t&oH zHI6kf$U7^7&y-r;vh#y)A@oc|&JC42S%z#W;MK;%FGP?m$3^&^N&CuO69ybpf6mc zcTF;TRnvfxWr+UGR&+9L8RuL2WSIA6$nEkBEKA;Jmc=k4-J+x5Cy>8TK1S%4&|9x;l;GZ8@A8Q=p3&0c36o5q+YX*VdryH~7Cr_O+{_*lc zIExOy`Ecpio-8#*a>Rzx`XE%u*t?d<-)m@SpeNcQZK6f&ZD6gRgfEfR)(~_|+T)`d zk=^d@xclI8PtQx3sy2Zy<d?VA`)mk=yGd`hn9yOLkQok&D<(Y_A7Ia>WjO6GlSz>l^)6@I`|gZ*X?(> zmpMqbQn?;{VdL8Y>;wFCPF-l_H-`8%Q%T1AOcZ;Mm-fuW8lVL#+Tb}M0B{sfj#k0%L~pkN0baNtHkk3^iXGS+V=lbw0-`rSs@TB+`v3;AFR ziTH|bReUBfQVTL@8f@}#0T&yrnfxHG^Dq#r!~wU!JqfMacR&sE%-7|(n~vs5m&4k( zzLEWgD*I*n#Wf9&ocAnVT}TmMSq+#l;Y65+2pB-NPcJ>)UU8I zX(03sP?96>2(dRz%fl7caJ>f|@lI{~fEws4uDK zmTl(3#_7}>e+)>e=P4*C=HZqnyJd<=!dr*(J>YXSf_CEWFZyDcsyT6peQ4xK14Gdo zbSFq?s)3`gZ5Db(Z1)}qm_cREj@WZlv6q+4`;Gu}WVztTmKyruLO~@%VSCDOQqPS3<;z&TIitT~0A@oeEb&xx zbXYMz4z1#Q_E7c?7Me$3t#6L@6iUyYeOVl<(F2bkA#>XDnY%cW5Z`H${ zKo$qdAs$6Q)ZyF2+;)b}hj*}en*Fa*PrbUS8_HU}1Ns0nBbIK0J3fH^86${UEPy5p z4h%bp{jfix8o|9J|0vHKnLbmlUSJXg^;`=Gd9|qcbgYQfjn`puEF%5B1KkAJ;O+)q zd|FCMQ7p@Ql26EEi}T?9wc`u}CWLTsyEnePK$r>*CiW0V);}D9<**xqqz3dJRc`ia zF&rX7pZ)a1=NaS#16C+F?&8*qO*_aVg3@9igNHGMv&KZ#{F@DAOOQQ;BYYudvCQ{m zMqC)`1s5q2#^l3U*J4sI2p(Nga8y2ACJWcGyl8hXbb-rzN?L8@vJB7u*C->z?kwMi1I z=?cEA_Pzd}YT+Me2)u&xZv)DYgvdWq-81}u(BndW(du)XR=pLU!N-28^wd9iRQ|&T zfPD|pD92tQ?C<|F!Gzu;LG)vv0r>H6eDW_7(iQG_ld07I;S2gFn+r%_n=IJl;UxI~ zgN22ylBQ=SJCKweYbk)x@y}``rtxdq|2XI0VypuC&zBj;`&(#--S zZj}3xsRZ(@4&49z?J~U#qUj6eaO0EzHuL_0ttDR{sG#K1NSG-U|4awjMD#&S@twnpIn??`kx;4FU)X0u-E=^tQNO=^2Ze4wnx^I|8BZ& zCij0axz+}VJx}H|HK=GH0Tc!O?SEUn3#EOB!atn0?R+;r6S7w&wvv;R;E|KV*pu)| z*01*l>|B&JkgFfk1+mQ#ekp-~nh}H+Qm282&>e)G5^Ty_vva_;IS*c|?pJxxj;W4Z zbpWR%;$8X*hm`w&Tj;r$e9JI77{z&`U_B0N$&@lGN#v9V`1r15XYX&hK}cQH#|&9X*h8W; z6GJ1ArL>f(Q>sFM85k8nz1@cw1W@*T!qljQ!Y5IfLHK^=RIh|E?ZPv1g zj2&D|Cf*)QDZjpa1XUtJV9!q!7k=2GtinY6g>GSxl{Sl7+02iNeMY?a^Emv?4uD_m zg@f{hT2ml5OLly$W>Xx!t2I$c;Gp2V_F*Mnm%G^FB-nJi7s*Gyh-CD`hY!VJjn@vg z83Y4&yv*fc)u;PCGnK`5r!QiNB2dmEkcbZ+|3N&sX8#~EV5qi4&h=6`J%Lp<^J0Qp z4t3(q6%_knP(vmOj`95<5hst^59e=_eNN%Th)_Igr4j-^1~wdKdx^ZT9TBSO#Gh-5 zudm((YE%r#)b_5i)2C0j6FR?95O7C}V%4}SM~AI&Mxt1x2e5%cOgU^`b77@oINoqi zpUZ)P#Sl65p|kFQp^XRTMD*V(#hReH@gtvl zmx6IYQ{a*x~cq^k} z(~{689Q?sAp@NfxHP|FdcMPk{xIYg;)$JF8oJ~W}qPPC%5oawBnv!&?_7{f;<#``g zs1Q_}{#>=5_HfY+tG`)XjIa?Bw1a{+D#hPNX9dLpnQuRDElwVay=S;|2#1QKC15_* zth@24&`MOCz(! z6kWNOa*Tf5x}Hz>o>W4CgKc(t+!sn5CpS0QwYKsUVzZ0a!^6Kk50aUbRZ6|U#E<)0 zVq)_r(snnq5?qArRY>*@&@Jw`tKT$atK=;3n<$v465lZ{jU4v%wf}Ht-LT|;lS0k- zjeP2#qZF~{HA@Fyzr#Nsz{l$Y)BlNyyLc80IJ%JY*VsUeUX(IevuKCHz%q(Wr~V-X z+s@0Z)&5z0sZ{#^cG%uOKe%x0RPX$^PcD&`|N9^;(#-!)wCRab3pR;Jq-Um(!9M}v zwY|(wo&bvowPnX_7vctm#XZbYiJ8A*w`VcRs;JpA5z{?!F)>FX`&zq`^&qx80T8hC+T^{rYT0v3ko+SVAXIdNoU@;{X6`8iGSYfXvX z@(>|)kbYi>)xf5crh6RgBA|?RLo+d!T)C_k@j#hX9fey$qUB&~AmQ~#tXT{a-S>Y@ zROdiV*aHqVew=z2%mDzYoj%NtjNTpAZjx1DB9Z1jUs;m!#Hawl`~3KE)ysSvk7GE9 z8(X@^D+Q-Pb~z(0BnGz+T{_A6>6ZC+V%S4SUVjT*EdQ#pbtxA=!`VB4Q@1R7?0%~w zj~%vslLX1qMBPd5@}keHCy8U{2x`#Y^*HqdRp56;HMSdk7h_n(G9l0NW)8W-saASr z_Q#dGx%rgyxPZf#dkef~3b}ktk8*AJqtM8t7eJcD3j9j_kVI*6E!mQ@ws~mjmHG2X zl5bvb$V< zTdfqgAj~b}BwX74_adDVB=Bw1MT4H0_W`cmw9o=t1nU0o;7*H zX-IbAZDoFfwwc|`BGc+bZzJ_p6gv1aJ4mmEDn{S)$5E(YGDL!>8|4=0NNm{rj$FGZ zFjjw~)8eqB7Z~8vqdyN)#6&i7KbObBvwfbdeTUe^gvFt$CP`1C^py+j83ZQk9?$yM z4wA|Ud@qcq+9nY}spiBQZ+1{dlt7p&xxgOz?dP&-PFi2Yg3^&lgbe)*v5d7`HNTab z;)`%Z_^JEek!85~B=3J`a3o1OcfVe;`O-dAEkB~XGw8`UYI0uT*Im`wz0 zT?Enr2Ho#CRXYGkwD$BI#_Y8Sp!i0p;lK)B%kBY;Ui`|B-z;i!5@@Z(&tO89>UUn) zGT+J9-}+-(?LP9I4WyY#0jr^ikKJD<5I+>YW!K-U8ttnvGGr>^wEbv^f1k%?Tet*A zsZ*6-sT#WBKX{!$TSONL#cBH&+oa{jmtUhiAGpK8`S67AXNpkD{j?jXBMA5=u*6+w z2}6L=X$ByMpo>tM?(WKra~+76Q@x-|De>jqDWxsc)Ou5^t+9$V2by!_z|McuU{F=@ z308<2@HUhAe(W$sbYwLO89THoD%Q?*Tcs`M6<t<<-<8X@G$AmD{CN3sy2f3uK6jiCQ}{yuWpktp;F%-uJn(je)AcJ&Ai4AW zU_$_bjMa-AVz~!yZf-=~Jo62~ofy?l$gn)2=9U1!6Nyd5FSX6NK! zQ&KzX$CR0X?XEF89tsX7*E=mJFo>JlEyZJ%tILtYcHJ!jyl?iCbqZF520E;v2_H{sSC={_?|AR%w$5l$f? zA#W=oRa;0|KBqr8F?f4);hYn5&?xaS@d4c9CtQlnoSmdRkjDmG{eW#k>Sa70`JBIc zG8=arL>@%7_B%F_l$pw=QkE&lUS{K(hlEYq>J5liL@#k5zN)P@Ok`cnF&TqfQz%L0 zH>lb4@mRr2PAcRi;b`Fn`w#j@@&J(?Hp8p$8-}@b<;XY}Je93Rszl4JF0bCSm%HoA zee2^GV28uHkBp&Y33sVu{S`Cv$p_f4=$;Tz6D^Gr0nm4TqP1NkG4PQ4uLus&o2Y`q zJK&~T*y-OQ3t_6TQ&svZo{2D{O)@AijMR9Zw7FgL%NCnO1fcIhH3f*?@Gcn%4hV0s zS8LYl+`p7;JvBbSfm7zHzkrvu3Z~bNt~>B_^Th}_C!d^~XdA*|1=Cu2smty@#WiyJ z)Zc1M^(zI2pOT`=*rX&}90*26b%lCDzd8FYe`V%IRYPKiOXrR`!{;?NdQ79p-LhO6 z0GMTEk7M%+{mr84&HtW=8K^lB+taN%b{RrTLXLa@C?)QJ*kYv_h+w4ieU4U>ls|Xz zwaub?GL#-kmOlKnJ>~Oy_?J(W3VeYtv=d47ba@@aNfdz0#|vZjiLVl~Uz+l%;928U zCtLhA!)-@@?FlEIYk0H%!*Oll)$hKM*b~@slHvE`X0Di}?9dZ?qY7Nb*n&y-?$U^w zM9y@77Eah_ZxUuHWu70o>*ZClBoPCbGu<$%g`Q0|%4H@-lP9G&P$myxIS%!YjMFhZ zKX``EJ)WqxGyb&>u}wxSbPaw|&k;xaSHFRv(+d|r9ypF*VFJG^b-YTH3X<#XAs+L@(Y0-)YL`iCm>=(cx*C7z`t1?RK5alaT~0l_BVxl+fwLdmh#i<$sQLx zSCqOkKT|fOzS_(Wt6ha9@WO(T~)fr-71ibICy(-Y0KEOI2ZFIY7HW(_Y>Si z`@NL-HQ#}Mh=7O923gpNVi^7)4cMf61tyak2$-R5W|@SUQzRm7ri}2^GN^w!z>D)j z4EN;3ZuqjkOwL8Fws@s6KRlU(_qEZ~D;dT=uXNS;8l<$)*y@ULy=s%ov$I{ZW{n3@ zYEf!?7)>hX+2s_^4L&e^Cnm6XhHPFDe={MnHs$O&5Igdb!ejnOF{ff%<+SZ4WW~rr zBfJbb#7`(@DLx(k#mzQ;)Mm^ex_=LFWj%slK)frHGB~x>-AAHE$}rac1J0mxrD1C} zY_(&>Uz}+N$+PFI&4--N zqU50Xap4z3p&zJ$(`8i@j+Dm8V-N*S>|q5Chdf3x7;RY&qayW{*zh|Y( zyDbZJTZ9(QZl@*(P(jS?3F^PlMal%=Q>n;sjWK=^FGWkq`*9Z>aa5YG>;Qkzb zY8SkXNHc@=Cg&8rf36cM-t$)fB(pG;unfWGf~i7HG3nAJ*c zaXxENDO6RMrUstF`ieAa$z>YuM7*H&ImBEAM58i+;~eDTm8bvS2|>}fYBzy8Jp}}# z+FMT3pvU^{+s{_{v#U${_e&U2|V9Y!(1URdvZtD8_P~V?fj;uoaQGr zuD5umJKuH%dm;RFMt)2+(gkl0H5AMD5w1h51q@#(g!gAC^pJ-a?ln-Od9gjr@9`7+-*^NmBf$~l}m&MSQ{OP1yiPe1$kK>m^PBgq-ZO8 z+S(XEtMPig*IEo-T=f0dkw$9SBT*!cLb&(fta!YM)TFq_50g z%OA`9_)Nkq$y8AI@ZkeVu1WSR9wL|%Yk|>^;M@^)(HI4#!L-;ycJZ5k<4m191o`xp zb<0q!Jb<;-stXLAjR0FHcft;X+H87+7cCM7um3B~6S=6L9INA5`lVYr(ze&0Igrtvnv#aKFqay(HFTQA6?B>J0k4x?@zMvwr&j9Ybs0WM1j3p@fT3=49BH!A5pU;QXS@39qY3k%so~swv!e$T56zg`A3}6u3S+5+a zm_&SNSdvIy!XMVZTPQi?mj<0k2eId1#UvErdlDTMR_8eCwLh-ixs%;7m|z}Z;>8Kh zMyN{8ZTU2VIb|lN}TGBNr(1f4G>uCb`{hIm)!^)zt$ z=k=(ba5Lw#KOZbSX*cFvgCZw7F*m>(nkoyO0t=G>IzS<;c8&XeFaWbC}Ad(NB z_wYrYG9Do#zX@f6KlD_0s!NMx>3owq)`}Vb;Ze++b7(K#c5qWl!hux1UB0_QLiTG{ zsJPZ;5;cdD?_Uz1$DF_UN}jWuda!2d*KJ-yz5{4h&-?t zX92WlO!9niT=nnwC34AzYpl8(#7#SI&CDg_{V`p~qPAZ)JKc7;lU6bJKxZx|H4oe2 z+*~S+I@fK3YRvA`q#B4roRN6O5W;P$zUzut#Fz_3GLzNtDSt%BbdBjo9CWrL9$;?0 z7wkjnj>(aP-bB(x{E^weFYL2xQ1KAIi~<<{?cV402D-0BRhWDJLtx33U0~J4&eJ|A zSg!G-=cdNH(w0Jf)uyyLHvP*3&t|79#Qr?g=QR19Y{EXAm)_(@Us3(Zz9hy&l982r zCNO>AimF;~-3KZAsJisdeSGfQQ)8#aKP3suBR^B2+bY|_?)RKrdu2B&!v*{+%+E{eUu=u z2un(Fb`!2zD?F(CO~5TZOd2({48UvX$~SQw0Ug)^-H}F$1Eekwy&DA*((_Z3laqc{ zgp#B%z>cZxurkbz=G_@fn`o~w+E;!dg?BQ)&ApZ_pVCdH5xgaHNP&fwQKDerW^ZC) zyk~cfo$pO$x96OKCM@bN9JaIhC?}}B-FHJs{o1D17%Q{167?*Z=TB+N*Dv?{{U0*Y zJTv3YZzm^B)Eu$h-x<7+Aa>VRwz-1kt!aesfVR39q!ukPbGPnTQs)1ojC4-bsP1Lv zBn*{G_s^<^oV?*hBWNje=n@<5@6*7*)G_tBLp9L}$Cd@@E40mJ?G1=xJN&$|rH^wlkZ*DKEgMPnokaD;%8;F?7s-5CPA%K1c<0nPVu zf(Wd<&21chHwLuc60^#a%rK_#MyOI$o3#+vZNag+H|);Cd!8)GA%VxOfFla*x;fRz zGhL?Lcbrj9W13O2lP6^L#z2>m*HuGZbU>bitQqAsriHKTbf>&N;xWZD%dkoVJ|MmV z_4|cSpAqqyG5shwS2UQFH2y<%aD3R4x195wvUW7@&$gX#fw~{TqHxqTuV-HC!tOf8S#vhkXZbO{Hzw#LK)v_cLIq1LzQf&S<;0{Z&QpDq;^TDhp03)!v8pz)Lq{ zPx*F2xs^WWK^bh=oUL2((H7~A9ICzw0|bnBIE(grmVcE$y0Ds7pqHy{Q=kjCy6Nz6 zoecjKISM#ndsmyl-Q`=h5)NRaTJyn>m zOsrCs(j(SY2c+M${$!lp*tcnZn)VsbTyAT{Ec`g-P8A1m#HwWO-miQtOQB$ydv%J6 zF-|?kGmrjE2SbM#AZal$H@;O8MRp>1WIC}>E(+Wyy~@;e?@7JpSNdOKJXoHE(%R~| z=-iCS0f?a$46LODr@Ah+4uqh9PY)9R69nrbN_B)@nVAxIT1s$!#*9n$%SMq@`7bC5 zNsxk+Vv@$urkC(~w6dwV@c3opYL^62ebb)fhcCT4fLKV2iol45B`m*`EkXZ{YCl8N zMTOBMl>Yiww}#0I#S*xEAau(~GnL9R%&(KlH1nuB@3OL)-S z`d_7pIPxAds0iM`)rxTvMgn=}yfa};#b+F<_?SIBH5wHp*2kOZZeMxzV|w7@i6Ej& zjkuYvTXPQir6#ol>Z+VBA{*jH&@i9ql$2KM@cewl%!6l?@iY=4& z#spl<+Qc6R^g7hH8s+GHfXh_j~6q0 zNoMMFq}_eFq2|>=nqM3vlk8ilCZZCypg1@;FG-`9Ey*jUH>Mh_z za(}nw;6`bus$GF2pICB^Ria~-m7nB5kojlUaoj_IjaK;xg5yF}YGbE*dpGkvjEFr$&5(M3hNV?=USXA6dJ(4h?#@UU@@s^3-yOV8=~jJX^U;G5g^YR>huJgdT`4!**>^kL z@R9b_$w_|Kys62Db*6bH3>_ukT_!Bw^iB+Cv~#$tk`RzpiT8Z2-$aV{K|j&lz2o|E zAbupcv)q_y845T?D>RyI#~={PWEOR+%IB?zgQi1%#^gYYdCT*;gz4y+meCmhwDlaZ z=Xf_(AF_S(3ia41gEwes5^@JS5Yuj=?Dkrf$f33hU|^YrLY1(m&+TN&c-GJkUvv9Z zwinZ}jN#@PK+xZg*U@`dgewwIh(bWpzpSXrz#F->DD^nX00EUqHwq?h2R|#nQ8jlx z<*6UG`NN9med8-0ea<6Tmv&&{78&dZ z63x0kOiL8`m737;y*1Tz4NO~$&z%#IcnRe+^k@63`T7P>Ey#?T86>kkDH$6j&VSp_dq-%ULNq{k2tpIPcg@%1(V1MveYtV~IuI00iEnk~`c75*I z9W*qP`%)`n@6v4ecI9p$Kmz~tR_>zk_rWnupW`sXCtZIvnN59FQJIbxocOU~+e}s5 ze%cheJvSfd{Sei+Ql?G&ovSbYSEQfNn>V@9mw4lie}355RTHT@sued61)wCr=hND>x;6GXzxwcGVM5c! zXrpz=4Xbc|wh6xBE4J5IP9&`oXWPC?<#|Mt4Y5@&Kmv9RlodPo2GFmB~@l8pKA z+WHf>!g6?e)2=xkeRi-csV?w?d*!1-CDfw7wbtW%SZbnL6* z11ReYTNDo7_h#`pMI1q#71VskHfcDeR5UoFx=lpDt;bI3rPydc$m?Oe*Z$}IpLed2 zr@}cq%|1@8kZGVRd9^IK2YW74fFOHgL8!eXrpLQsG7<}3xgoMbv$q0IxjJ$ShiR5@ zJA_5nP{RWH-f`HjA33#GvhAIeROj8rW_dI!dYKdRYA@d#T3OQm zM4xfuL+Sye)nYM3QjC8+FdW{S=El|EoiuV!7ibjpk_y?j-?lk}Ug&8l12m}i(`xn{ z5QuL(^gT<%b^DJC974XAg7;XSYPOuGduf&zVmlT1XW zSP#M``&nvUzjWH@{#82Ts=UmGho9AK{UE+Yz92if`{Xn!_u5+J+X*vNCmA>Lem1Qj z3szv=-9URDqh=Mfm$<5@&V0cAhGsK-C2;6%rKUbaEUv*Y{Jhd?f3=_%c3bp02RQ@qaV~e5hu!JN1`fDTki8aJSgLT2-M_GOTW5& zct3HGVQCrnQ9U6m2E!3=HeYoVm)IEu)o66*-o2xM^8ayr{Wkt^;Z#HUVzrIsO((Kt zBYWZX)1^N4x{^z4gp+^w&kci2` zf5Wj)EtQf{t{ak8KX8b6&Z`-8F(=P)O_)^h;JumDwv07TeOVOfX9lJoucMt&tz@w= z3v^0LSlxKp=j+Xn^9en(k7fs^>Q43z-{HOPN8_~lPdfKaMYZb4VNaJlTJN4ZTHpOE z<-$Qh%Cc3GTQ-I2b~)e8-W>1>qXwf(m1wi(fWmyKyPcF}aat3QMroUBkC<1KW~5<`r)gg! z+qScQ-21LI9N{<9Ji08@5l9__%etc7Z}VH|n%pD?Ti-^WeVI)>%iuue3O6cmR+ZPP zUlYw_y6GY|{C|=&PplYl9JxCt+tLnnNySH8NX_fhVI#3~_8i^yOyTRPo}Vf43A3j; zfKxNKru|L7d8GKTsa&)j%iG(dQLM_@XPE4hkDs!yGwUe1zFhdie1a%V^fF6P|FnZf zf(molhpSpv$#_$8Y6gm|*!yHddxjT5r&sDP^cEbtf*_#JRz**1mQ?R;&NUr{^}6Y3 zKPg{`(Q^beS4D4uqU;%p@|7CTbf{8dHtrSBDTzR9n0#(KR9JNc@87@Qfvu5y1N{bn zoe;hYgrO8A6QpPNHM9aabMIYuN0RF<7b3>+#D{y%H08j3N(YdPZ!G6OZrhw~<T13-Nr^J_>y50m7=04zhM%t+-=FPe^@-gE)(%hqt8obUXiT?kjqi2Yi2-Qv_fC z^i({ddDS}>rCU=RA2*4`^UfVTx3 zWts5>M6r6EvV5ge4DOngGbai9AS@}Fvufqa=aVBggn_ku#eCM}Ao{2H@e$@_!vMl? z3DCwP?C0k=(R^jwUziLIW|onDb+6qwQ-@YnmmJrjC*G@Lb7tQnIq1m7{lqJ3Ilq6y zR)w#Qlt)r#o^M=(u*}&~J3{CC6!5ZueS+NbOuW=HRr~gFZ_|O}|6s8Cij#!?K(#f~ zo_s@~sQPcGD{4eK=%6hrzZj~td3bmkd`(c*7X;j}4{EcqmTlFA3uVO!?nT7EIs#zm z6I69zTv_M6F3rx3qtC2*2WX<2WAe!z_wU^+8yJi%m4<(-$RgZ71ds(+Aw>vJ7oX!_ z-#w}oiWtqxs@Ync8PZ7SN@==Pj?-_kXo*dW+?<$K7^U#mt(#3EL3H)2Bze#jkxvT$ zMh7GE0f8c7(T@O#b9O2Qb%9K4U-ILK&|5&-$p00WmUad%>!&RuSyEq~J{>=tJ8yLh zF5m(Diz*Z0mBbA9BbX)aNU06$>YkQsdA)x9T8Hr`cPP1K`q6!hx4yRA`;8U95sGLi zyF{W0JOs-IzpLT~DpW;8fr9VO&}bsp{kwAQC`Oz&!7@N5%5Q~cu94v5M{!VFm?6V2 zMFIK{)W{Jm$x$E|%2FM_r4v}HNAVlRthYa?Eo=ttf0vgjxikYcp$nA*)Az%i0oh}-Ny{-RQ)fg#h`cadUfS9HIWpuFV=z?rX zY}pd~<3&!JB|i&8q4>$cjAyVEE=K9|(>9&ggpuE=y`deMrV6O;KYxEGcj&%+@m--p zg_1ckX+~0E4+(CC2*)r)gw&tt2mh|d37X1Dfz_;mz@i!@L!$rSM%Y1GkpRIp27*^R1U_76RBV4|cp=Iw#nD9^*wOgm=Hp>YE30A% z9s_fY$Q#(Y#2s)fLczNOkGT~#Yha~#V1ZblElwiZZWsV*;N57o+*ndVER_Q{wTkGb zZXX7b!V&dp@LJCj;Y)BUq_W5S_(;hOJYKRFGQuq(p?QI^Ui@H@tKe8f=-O2b&~*U9tP*=>fO3>ZYV9;cnKnlG}f|2ihuF*O@aARiSY4frs+%x~UWuairbr6QRCf$Ci#t>%`372W|VgAd#a#E1_!=<^B9m&iS>S=_Mx z(5rzL5P+#$IF%I*j?<&!eApwUlcfV^YOUq`zkx<9xV8|RhE8I22U7c|+AJ~>^vkLl zHK5PHBkCiMVMJVRo+YQ}13HA<%*+f?j1eY)&1O-}VowpQF$QdGTh!Inxj4ulRW5A~ z2MB8Z4wZj$l9{57$K#q!dj$@XUO?~N0YyoIfY5zv68K%CRi}juae@!FRtd~XGQ_Ry z2z?m9S$}FK7$n+7%F;g+|K1*+25N+N6yU67Ml%N&-ad2$^uiL7gak(e;%_D!Z4i)1 zUm35Lq`Uo-tb<_cOtND=xS0eaTzPJFYr$z`iTmQftqXFW+>6tzFvb=U@YK6D1f~!1 zbR@0lh+m#pxg6jY+@C8{38knc+CSmV0p*o6RyElL$Ug~-9bQ3hpcWL7-CbSqIGR8-Jgw111t`{U7rxCwdr`*u z=YTG~e1Rgs1x5^|Gw&Rs-)YuDvGCLQEvon8Xvmg99Mc!t=9u&fuFF&z^}p^A&eh!zlxQ;r{vy+h{@UMe)M~t4 z>x;WF35P0942WUvf2ypkT)7o73nQe764ll`2VdCcBHBh-jM37nTm~qFV>h!l)Kx>I z!v`Ud9cPyIKhWG<)nbGB7C4DF8JSUnK|G8w*aK(zS%}DJx@L}7Vhb&|dJd67Mm1XU zF(4%8?TUlOr(c!2M{V?1|G1lU63C_QSJ|owX9%E9| z4QLVzT*4&-vgdSo16VEh?%!_<9Kgi%A@FzLx((I3!lHQ=IE$tD1;_}pEp)u1_#y7e zugnygv&4KNOP)xDn6gfzFjOM&rLHUxI~%iQj=y49sqT?3Py}VF1yLHQT|f9@8H)l~ zRv};Fm5;qI)wMsKP^N<-oH}6un*=9=uvbDFqkzThPW%y`Y-KF7oP7(W64Sfz5XEFI z^VVi^ek3gR0&&>>q9d=sl~&(w4*6*>3~=GoBnAnFCOAlgzq=U2z+otcHN+usG<`|O zm}-CMF(j{Ey?W2LMw9z+$&Qq`CQ{%;UQCP{3XUgIPJRhJbsygq`TiY_ce0tQs_iF1 zuQAi&ffB6={jnG&zQ}jU8AggKH3luu3yZ1=Q(PwPQ1vp5#ntN2R z)WItFIHD^LL4&w6cdSTht?p-rTr?uMCgIhcR1wu>0cva10VQ}A-3YQkuo>0D(6=H6 z+e5KwIlh!4cu}7DV_){I*N8SA&7QQI$(Xz~A}rWCf&AxFRzy7!HbXP!NoZKruL#mZ zv#!tnVD2ro4KpOX3}k`Gi_ET?kaElr&vtz#oO@cTICni!D`T9VBL{YW(f?);)BM{F zHWl;*kB1KBZfZ}h+`eH7>MIF&`yYs}w!8jl6CFE=WZ`!a&>KNk{BAKSUW4a;#}Jc5 zg-~lxVUl4y**JBp51_sz>X}H@lUZeFI+9roCRALHbN5uOzOv*(kuTU`F@s?7d^7KF zd9&2T;CtpJLhgce(-$d=xKh+9R|K>~fu;c}^icV+mLfv`EOFHp=QRKMVO2w}ZB|(z zzr0Ma01HwdEx-EZ2cWgm=brcqlag*&e$}KAO=JY`V(i5iLA7qMIQxk>qVV3?WWKPD z_ZL+ZcMyjp?O>|{?=RFxudmdWJPS4~16k>$NDvr~AW`%LJ3yG=XPj{RA)0Y{H`Ne9 z+jiLGa^n4JWNB?R{T}l?q2)QJIeC#5iy?%t(?O#9`SNr#T!oM`x|*Ln-U1GzuEc%a zPVuu!*CkQ5EDwGO#h#w{I^qtDDAyCiN16nbADu#cQpU}8BdVq;MVA2zSJUD-?AApq zTT-^GoW*v^8G^kqDI^(te^p>hMgrA_)?fUUI99S37J;zd61Vz*;pHJ#-piDZJ`<+- z)atNFg1I$(z_g6d()A}QCZDU`zu`;9p)1>b>nofoZ^h|eHjV}@4SFYiiI2LW%!uK{ zGf6Px@a3I}+jH`S?q@;8e)jI2aLt=+E$6d1Ia!BsG&WtSBVuuRrn0Kna6RFYg~Ge9 z?Z@8h$F))1edF7oT+q#b8RsA&G7Tq>4S3Zrn%PJ%#ua<>kpu4I$LS~rYDnK~+26Sqb zjK}28%@*Zl!cJ7&I_`*grxw_C6qb~W+7U%y z!uW4$3lmW$g_QDZRK!$Nnk>;^kc?|^E?DFIk@p}3AB^}!M@v9=NkoA z9&{}1&nZ7qzSbz5HrLtlcB(BY?Z>?%+hC~oL^C!;;h^g}Jmk?R9QHGki7WAhh!+3H zK$q@5vVbw(yHZCf*W&tLmpxDlG2JLZ# zS&k;*;e>jTAj1rA%~WXT}>kDtped@fXGi9$|sK7veP;g>}_UY~O<1Hk4Gwz&Gp zr53(PZa?u!`0szQcm_~VIC;OQ#c$4jk#tk}Ifl4XxX){;Ysk{qvapKz7|M;=k$%St)|IeR`loS;2ILIhUtdxW={B7N{^r&kt zkar(y_+JD2AH(F`uD4KvKX~9j?$6P>+vD)JT-;!c?SG$%3&SBoX6Wy&|M4g8r)$YN z3+4NJxgC@ve;Il)!XWT8JV}WE(80C;^<|4N4JJM(X}*OKc_e_yGG?ja8ktMv0_QnR zw`e|;DE64bFVC<4dv9C%rNn^}+{%}wJ}>0$lVLl!Z*<{v|KsO7SoSb0gnH8~Yz!g` z@Be%_T9ay^Ps6kAhqoV)*xhlg;2k4265w791^~UNh5&32HArJCO;qUDnRjd*W@?o zNqEw}nU?ARCP7HYI|g$!ODJ}O)?!3y2iLe>Mk z-*45RAFV-Zni+!I!fTIdcS3z&5eG`E$-8B%HG*c0az-x)-fq-w5S07u(CT^52nVPA z`n>{tun^gdDu$6k`uE#|;3^e5EL+j)n>V9L8ZWOq$q`fo+f+^jW;?QalO%5^AnX{d zN0B7?9q8|07)i`={`BvTwZm7`*fW2HS;7e5Mr?eV6C|DV>+(rsLz7QC6!KAWPbv%i zEv875hjLV#m#p9uM6)ORGkX#&!7sXjAh`-Byd@CWWZMfWIPSJ1D=^5$tu?gK%}{61j>e_D+T>`^pT#82yuCkPSgoO3W4svh8 zF3%xDRk5EPZKyo<&G-!O*~`sT+JReO#`Y8*@r4+{wECMtW)=Jp4fhQE<5sP9V#6Gvw@FPB<}_RT^1oN>*J7Z2(##3X~N2jc!LZuzzVlarAQ)fnV$y5_z82GXlc~i}E-3K#O4FUScJfU!XOJ)mvk=t5 zK~C;+)9UjAFzMZb5zzw#$N~E#8nPsB(rvB%1CdH0+)wwuRbgkxJatkcufCRSY7U1Y zpFaGp(ab7=1SPo}<(XI&L6|xlWO9GJfwcJ09*LVMxtX9#LIJ)d?@OWhPhxn{a9&SW zCxxqmk6~Rtn|Od?_3h-Cv8)*44ten0!mQtqS+Bsr+^4TdC=UUybV9i$9g~xAgH{Ub zBmKc{&H_J&!kVVCwH#5X&$i?F5_2`l^!r9uwWfY+m zG3pC%DDL^t87*RKbcwyPasUZYYC8Y54-e|i>r|6ElN_rE3 z@Upyp(8Hys`EnZH#w0bcVF_=06`P>cUsyJD3@D|+?rhRt`-DAax;8|Gp(W-4Jo-EF z!Yl0Nx`gK6mfgg`dzpAkQJj}BU1iO>8m36DUNXm7rYB+l6c>X)8_zg6)?`$;@kH-t z%plbqJ=#^P@1?_?^ z=d8f|%y1Q8l{hY;S}`K{4g-#P)bh^{i5{e~{A7RG5X75@m3pkFNY;-ANQa7s=v;hy z^KXNl9^0R10a-~7QBXCn<@@_URNE|5>2ji= zAmFyshRk@XJ-3t@fTV3vmzm(*cw*vFh^^Y=jW@$v@C{J`v-=wpsyP8X?+J08J|1k{ z8f5urP62x5q>ev_US!9QPxtY5+|04HQnnxcbo6L@=g+}W&y-PVJA*HQ)F7~{MebG*uLygK@ZBMkd@=J}%o+>m4QiaGq z_bm@O_F`D>=0Jc^*QZs!vQJh&mUiY~E_bPI;(0ndX0T;Nh2EZ341Y7D%l{pl?s{^(na?iOyo+x#L;1#s z2S%}}@!Mn@*p6H1=AE5xd0nV97=3u73yg0HGh~@QnK7Ibb@q|I9v)2F_Q1=+B|g26 zI`2f=zy;pKlw1}mleE#q+J@j~;xzjX4f1L|*tNQc$&s5(N>UqxBWM$pD9^H0KJFKv=)B9OR>#_N+Qp+D6x>Zr;v-SF2 zor9y^Rwe^yy~>4t9aAnCWbUKZ4$6@YwEEpw z+xS&K@u|ztc`J@{dGjJtbH5+&p2?ffsqYFjZQQH4xvMizV28!*>_ex?*tlxWgxtCG zH^osw*XzFXI*HO**-)u5o~3EF&l*Y-97>+}*e#Sgx?)rRj>3>6{Xlb>5WeB|`S`EO zqm89E+WeyBltVUqOlnl+GWf-sEv+)UR=Oxd*B zozmf5k6dmzOZyja9Mj#}W6000Rj!oT@-jL^h-F=CZ=Qm4Mh{1<=*LGLB0pqL8p)~H zms*%~zAHZ3?q9NO=Dmeo$SRY}xTb@TgkE1xF;bX|zP7eKf}+wsi0^2#Rn(17BolS` zu&s#YdSdpF1dZLq&xVZFtiCkm%b&S~3~wnaD$?Bgd}vb zlaT!QX!J(jYcq^(lQcU_M*p=4@AQ_Vl@ik;yG|`6J!QzPifpGjkEod`tHn zz1I3Iqv!6+stl9-4ZbO{+bt(+=oDPC=$=#C=Z!zk7;%fu*NdGS>1wI9XQ?#EYR|rO z=pJA3-RYF4f?4T~FWh8GAL&ZYM>|d|M~H=_(Oo|O(MDSTRR#%J{h55fW5N%yNh_#-?9YU zhd;NGbCTkG!0)=>&6A4o@~w>O!QF3VyYVT-u&XwyJM13!XAuXaWD?#Zvw z4WIQ^(`8+EQ=0@HFKg zlaDHh<#1#Bm3z&k*-ra1(u8oPOYAMZg_8^|Z0a0VJ%wE(^`R!*dMSTC>jsRL35kv= zUMLp%a`%*!+|#aZxz1> zjpULr`smen4$n{Ggxi;@9&gSYB+&i%kgu%&i0!0xUuj+J8U@usF{!`_)tQiaq)qd; z4Q5P!%$W>oKF)9H7uctNziaMi|85~W375W*%xkG*hcD^Xq*i^6O)53)`1tDNY<|`S zUfBVrPaQRtU0O98^)Ch)FA9yF1xqX!k^;gvyN!ehvtvyh>+ZDM0wjD>ds+Dt^1tMt<*?QW++cDWqJQm= zFCyO+RsRMD7%~+hRFWEYb7@KS=1%wTNHrQN&4r}M5}!*Bl{nt7aeo8vmEN@ZnJ%G1 zY`%fBGo|WtoF~Tc77E$Yziq`h=3R%qVig$OPNVX#nfmO}i|h@Qj)mB_X@Rjjm9l5C zHe0s26j6oeT=nPG50uufc$ImEZHKz7|1l2rM5d?n1`>hjPS<*Gp8sHGLf*MsQzBUg zc8ZGZ&3A67+!-2-pl&*5E?P1rp;bQf@}rgAq(47T!`EFXuHUcEwV2cnkK=Nx`l8!$ zMx9qb%~(siI?QU)!ou#kQrc%Te~$NNOFP_TRcf7Dz$z0G7V{yMJDekqDxzJAq3UEn z3vmddF)iAs_(-axKpZ0byY)x|Z+s3tRl2=M{I1`KnBQ?0cQd4q3MFacrE+63*hWKY zj$SNu=E&V09cU`C(_Nx4Z?ZE$fmQNu;H@$gi#5{JnlCA+K3io%@@Jydx$*4M<>F3z zjIQG4&rsXF`I6{Wom{LHbp1!@iW}7mErov3&3nUJLN*^xHQZ^_pCe9VJ@NHTcW0jC zHaFU>Ew;yU(l+hziB(g!aD8@JZ)cmJ9}Qi0p7A3l;h*jM*N%x$HxOL<)4R`lLjK$@ z1-!2Ms@_)oB`x8{KCIn#;Bb5j&WG929j5{hX!b??tgYUxNsMB)2S~ENJ%GCCu4CVC z$w$)5cyc}&L(R&dKGj>7%brr!6e;$_l_o=er$`*=JFz>{VnD5fVO89O6_>X6Ce6AO z@$f^w$8IVK+E0!6Qw=3tij6w@atv=p$M*MCz$ zeY5r0taWYC+{+VkSrY!-@l#op(gC#w%84ZvS-zWMnf5%sUYjH`8FGtD>*Du8!JF1r zm!)HUMt50Pj4)b~y4-oo{xW;-O$LF;`V`YjF{5{7&CT3($wsYMtLBy7?=Ml@-*? z^|<&kifN?rm08@EOe-J-T?lGFm|F)=R@yYAXji)OB-#IVO=PlDe<{5yGoB+xq?D0ngbJHEebX^v||HwPs>Z1$sQcnw=Njn~YX>`|L zF(Pl5pTxr)TuWB>fZcdj9Fz3g)`(CMl2B6fy=0Q2ZZ&xSqLzg09eWyIU9w`fdBq?v zCAT5uVkI-WI}Bd0lOFmQWsU+<$!N6=Y(@g&)XtleJo!?EtRlZ|(f*w`dqveXYuHsw zdl=ac3fcEQ9~$*Zztc6DqI)U5PglOQE8UE<;LCM^_mr4+_JvVY_+P2V>`3Jwkh5MnjBJ$nkGHQ2>QhVs36hn z-~Zz^$}2XFk}OCkrMB+VE3KRQmXBfooOrpn4Uy4uIlx!bWXw}jN0uILH-MbwZF3Gj zRE(3LC!iXUq1telq)>QoQK2aJ!&DPJxQ2r`&fXkkBETdZO;P%`n{t+!4l2~xgrBI3 zuVVJ7E|0`1l$gqXELFBP|JbTh9awzP{A28S>#H+2LrXit3}0|gh}-;leo#|Sk-X=_S0=BxarT!S4_{J=-@6%XV>VUfNYkd5E)BL3 z$5mdDMc#@(Q<&h&`UBjBtv3%Y8Y?%5p|&6&k_}cvn@7cMs1jP(QO2Ye@$t>q>l`a3 zJ^=9(@svD~`BDRt$^w>lxVMHtzhyZr(6OpD=W_!rh{hwbKp8Rx>#HyIG$fdF@9Eog zVa$uBeYDcsD=ZIjtix{!21TP4d&;vO4=!Dow5i?P@Y^3Wb)UAzpw7*8KUQ{i&!UXh zdk(iKo-SJ{FYdntk5#FsXr~&v+{~nvVkXG#od-&bO0x-p)%-ENt2cHn)rTs2l&MIm z?1ynvy&UmFV2!8ef+zc)9TW*8*Fg|^SttU6w6|HwP$lqSdn)hF5MEw)MX3$yW=4Ef zi3T&wvo=`LmNE6~-sVJ3@3D>cmd)FzqHJ5c=7;ijm>aTvI+C{Thj$BLs9QU3ODbh< zD9ePYC`c;gwu!9H{GqfYX=yYypCMgB6?hOLaF>U);Pc(wwugIEOqS{Ip?t{}yys#D zCHZV?`26mI-p-!n<=bN)SEVA%kwFH0)A2?sLg02QM*Fpo^#iUUbG6c7gh5s2#pB9# z7P_6Li>L3uw;iSv)R~NI+lm?2Dxm%%Xc+(W)>Tj`MBGm>x@S1B9{3D5OvgO}3rQN#Cb@*vh47x_819eTE`!r>7X0J`VIYEQjVXj#;!&gsVty0_PG}%}{ z-WU*LFIrS)x_y6bQPg9S1FQXWdG;oi(7P%VwC~M!_rR5N+g}tx`~kS=&Lfx?v#?Vc>gV+wXFVRB6;fpF&wMW1V+v$9DGGd(bC5OE&*YjMpY?v?@_@g`j0*_eIx1t z<^u$0a}O7<)W!GA4>jm}L~fS(oNjn3FHG(syLM8Pr`{C}72CSs`Fd|A;Sg!UC7Z~i z8*s&vs?^;L1v$h0$6HGMM@qihGqB_m@6$UyJ9X^E3#~Sd{6(_m3-7Lug6mB(#=9d_ z6XzpE@Q%g%V)e~c#~I6mhemKglhd8eMBA`+X7-7VT59z#m~zYKt2&}A(fpQj%Nf~Y zV=0;u+^QWMPX?P_E!L)9P?BwRPSHp{_k=UBPIYb_tmJ1W4Qeu99{+XUfthcG`T7;u zzf%5G+rB)*B%WCL(Al#_7LVI?bS!Rr%x1CwVsYSWv3v(+*z8B))SB8QhpKJYihhl| zoy}J!tG|SlfG+sE@hjWqMV<2Bh4E?tYft9>NZ)<^4;^8b!2wczd)df`@>Mqx#Ao|jAEWxj-ynh4qQJu^A^HDtoW7XL= z33dDt^F@aQh&7L-nDQQFE6$ATKNl|2Hx!hvNi+@aRE<)Oz0xzZ-+LJXbbnn-&HbfV zMtaUR$LvSS+=8rtyQW*h_d7Wzs>W-wqth-?2hU?kDEfyvSnL00{n2MTDJn|)Z*NB- z_NpLF9NU@C=K644gvB$L;o$v^sD^pyOE@ zPRYj#J}ywoSgsRF_x^Cy?u=W>c&ao=j$J%l!3~r@5^G)!SurmV5de47F~GWUby;|y zUzq3+2W*wtcOTzNQlbM67On6HNo;YYMST`Dm(@n*I4&>}o$3Rka<=s>22wnnRk7dd(p&&Z#TMvJbAeSrCNF;|1OQ z$-FEqCa6#i%gL|nVn*2JO!yN4)L8~%#|Dguw6F@ZH>Y;-MV|-OvZNaaA6=rpzTO7n z^$4*47@Nhqi~x%#jM8D?4?lUy`?@sGxq;$6?++V*OC^x$G3VvzZUtg-V`EepH`adc zszwSBBlW>NkudEd!4#qh7Zgu*mYoKbRSaz&v7g3oaC&|5u7G!+J9zs!#T2h%t5Lep zOCkk{=+1l59E*8Ge0iroS>g_CZ!eYxib3NZZO1paeo{`YW~bD!k#nVOt51Wh@TJ8D z=$9pYc>xk{Ooh}o*ULYGTaq{qkE@H`BX*2pWxxyqhtUhb)*>vd+R7iBZ16neymDYw zKOytETpdaM&Whvo8owEjZ8@JlTv8pK>lbcO$*pDjt4w0*SHrrTb8@zPiaBQV+t{?^ zJ&%vvQ{63o6jw$rd`VL7GVBd_2Qb0L$Hz~Nc&WMbPrl20j*^~x15Oy7IEG-mwAJCY z-ACHocX<5b8o@)f;tkTw!3689(uqPJIas|!*ucnVk2~;lLiFZN8=rDL_LOtxX*q8? zjolEU^t`>$&2+$28H$Ji+=iEx#FN2>lth1CEZOWm{jsm4;^WE7xKkOkuObBXLhf|J zN=i=!xt&8BD71|!N~%;CnSbVFi>BqtpRML~Y=c!+hFDZsTM{40BJNFaz8DB0cUSwb zIC75=$gvYTa;`!T(|u;a&B{x5ZBPbH!q@!TRK6yN`cx4^Y#p=+C@I3WpudI&s3mNW zrS^XAb*#s~R>^5FJiJei(y0$tfo_-LF8CwHR{r*-uv+pV%BTcD-AZ~Bywi~!vfT%p z1GJygl^4H2X+b(fHoiqm7k71;ycn?5bW2myZTfVwc z18VrtL)I5o^(p9wtUWx-N-7Qxxl`~q;yFMRRaWh@-tX7vsTl}J80t0SmR<-2$B_U) zEL#z{!{WE0?1;ee1g=o3FdJ%c*-fI7jnbY&l}@0(oPu<=2em8Fe{HNTO+s7s;Xi@H zmIMvnD?Ivdkf)@2A_t6?_y5k|zo_v#Hm_F)5iClatbnF;Wg z%*rzX`{bPRd+5uNH{|ItKi@|``ECUtl`^u?NHSpK>11p>Y3T{Hb`?6ML9yXA1tcjtewAsk~CpGB*CY%ZbLOh@-cW76#$$Xv{xHQe&&g|WV9zx2L?m;W7eA&k;nlO zGzYc9loCY%p9c@uES?+e)S;iJ!D&^hO#^`W6(o$8AkBDTgVvRc6hd5Ez=-4~QVc>Zq4_L{FAUF-%kN7UAH9r{OJle93 zzSDeH*D?q!-rqgeRNLzZB_xLm+jMB1;0@J=AYL9NN)BVMkSsx?S8b27fwMV$q5yKX zRat*~0|2`K$t zzf<`()WsrE zTXydnZ|t5W?8a#FYQ zDlA(_yOC}xYyK)QP;ic2sDwJ^Jm@5g2YCTIPL37ZTH>4@->%mMC95Tz9=RM$zW;k5 z6u^f|k_!r#pJ&X6(pqKVHZ{)h_JC5?o*$mwhC=rVpYS-cDfeYI<8@!c}i zqAS9BDqzvg?D_;Jjh)PokJ`TrM#8Ct^^7)YpDES3S^?7}ai4Vk0Icp|(r=)Nvu|rR z!b+wiJs5(2mO=YNd|L;XJwIIDYRLZB0$m)6r--znEYQi4A0|d^f0SXF^81u%UVaiv zXnT(PBsEi^3i^PetE#B(lirXClM8%r#!Iy3fgsojXUI{J>}rzbLQNmyISy)Td+T%= zu`MLOZ3MiI4=MFfAtF=y`DU7Rg%+DzV&Gg|PcwCG1XFM*3sfI`MSWgp*Cwqu!vT?U zq&J^sYJIGN)}X{CQNECpfmSBmO&)g4l&fWcTjhfar5ZxGg64h@N>xKFQqcSfGGZZD zMbV2LL{i?$DqqC^-Vb$M&K612L$sD^Kbws zy291t4re?Ut?9)3yKX(IaQKG%gH^V!A4nAnh>eVU8BoGJhHeufNnC`AYF5rYpbHec zKMfa_<%eRctM6`EfIHa>hPw%gl(^qMm#6B4& zvf#k#)8NHLOETLC8_^&Qv#KIONc2-tlftAO?i#3mxaN0p@I+^az&RyVm{vSrmnU}_>Ohv0Q`ZL>+8^<{8U#B{F1L_fu)K|B;QYVB1g*t2$CuK&24|J<v_+h;P4PZDd5QwN4se=3anf-R>0sjri8mm8&rKVv;mJg9E^v z8LBM7Sx<{4c-4z?&4WwJ+&)i;0)YY1UbH((4k7^y7(_25E_DZ!D!Z~HbgDLe1I79h z4R?B>m|iRrCf>bgZe@pB1UOm;-TEQYGNLG1`Z=$x-JrGm6GoL6+x6MA386aM<*RTo z84XP>unJZRCP#LNAf7Abx}_Dg7vNq^(3QBcf12ldm8Zx1fiU~`R`rANoMBA4_3y>M zm+?yCPI-6vk-;8e4t7$~=4$R{eX8?UsjMjqyPU6PW)8a^9=@%6wqhw6mwd^`7XTb> z8a622fGWg6h@DW95cy-NaisYUKh$)P;lV0DI>I4i7-PUI%r_W1mn+cGZ>rl%eb%9^ z&!gcIAxTa@j2b}pqvA1b^wgh`mZfuS`Y$;XwijB<4iLac$cS580D7y=^Kf>j0QGcPAf zn+%Q#D?@rla&!8P6`Hu`)=%py%}omg38l)K81kjEc3n_2kP_igeqvy#pC;=JH-zMp ztg5Gnnhzz;3!~oXwV7&dqyAiDcl&b#MIo(4waiBZ!o&v&5ug90nVm4yTKr)&H!3PO zGD_ioo2>q}r8T0#%R@eWk|t{k9kXXn;gPWQVGUs-!9cKMb%oM}XHe6-@Y^+`V>6rl z#0H-1DQ!3qXsVmJ?5Z}#}E=sZM!H&Bwz^E7* z-x^wYHMlM{nXXNEHhxT|aBx?+iJ>X*hE2nn$y2`=BrDs$PHvhBd?R!qORJ4DNsF~)o;GKy zhGU9^sz*e7@L^JG$8t(0GGjJE zz=>Rgi?tPa@QHWPN|A{S`lgm(|N%Oh`LCn2f`mA`f5%UeO&PN_Bx0WY@OSx+g!DC zi(n_SOIo)xPmb3)9zsD-1cz4xhV@MAPxhkMAN4x8)QF}>ZU0rYR=-09OKyNzw#Xuq z3WUy(JH0da!M51ey%+ZZ6lT#=5*(X~+po?FxmftsvbQj?A;Pvh((E6PnGWWR6;e zZ-w%O&vgZMx#*Sey$oThq3^%|L_jeUZW4C_FADi6rCrzQMcJO1XYv@tZQ&;ixZo+Z5`f(VAzJ zaXu~SLS~G9RI9F2$a2L2B0=*WkKuPyT3uMS@?g6%NcS%-)eF|R@glUfy;>*9fYfHwurhe8CwmXBknvDkDbVJzc2uK%+PSu1*>6QTy$det*G$Q z)JrW;f~ruL@4=Ovwl@aj`AHDJI4r8oKx74yoy)ShnF@-c`jYl_$r8E1^j3i@ZlBp0 z1eEpUf+J`*NL(#Uf_hdb@Tn)yS=E=H`B5IpDAaxFYgh;M0FGl04e5{lKu6)&2C}PT zlkv@GpSbUXVBvc{@mp~!d(jGaAw>$txSUM!Y}LP)%IQSdJT}*D8`&5db}PwpM;^!l zqP;yTh+Y`OrZP_2O{QE|E1Tdh&PtFBKX}ebfg*gxt!scHjCGniCb*eGaPM_{zt<5Avu9`g1f{a03iMXvG zXOrO5Y1(tVEOiA8*9Y;c&@xJ$Uw?ywjMLL_q*DghM&&4gKaD}Pfya};Sdgb6SFoCL zeZfc}TNoItKJ)i5V+ z^dtBmdIFHYvp3M?CGuh9fHAsf4JO7YK&Qkvc&5u5*bZ;FC7N*n4Vp615=fYYAgd)Yp~qFRog03E9kEJw(Z*gKoOdTdP8hhF9U8|{3e{!23e}0;0|~7h9IDc4*~!R zXo`fpfH87=n!`#>jOcYP^7RNlWmpSJ~I*Xa^Q!BH5zDo12tf#aGj8Spi#DwWjR6JS=$0t%T;Mt%T-b zEv)xgBAXRxz^CXd&S&uOVKD%DlDYvLc@HeuJIwsI;2 z!k#Va`|Q;g${@)wW$ILDq%f``LJgb(RwLeH&98glR(}EJbu<-QsY_!Tq?=VJ+_S%j zg2fh4svyXO7$AH0a0gv}_<@=YOkEGT?al4+-g8|;;J>iUco3x0CNZ50u!^a#Wrzhf5RuRyz?xkU<`vGjLQxO1D@ZMX`|JW(TMVD4Cqf23= zVjb`5td7N~d%G|X0zLy@CXfFyiv8J-SvUC~-28u!-)y}qFf;$>U+mke@y%U1zdftH Tot+ Date: Thu, 26 Feb 2026 21:26:54 -0600 Subject: [PATCH 9/9] docs(ordering-matters): document workflow, exports, and analysis usage --- demos/ordering-matters/README.md | 122 +++++++++++++++++-------------- 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/demos/ordering-matters/README.md b/demos/ordering-matters/README.md index c23c485..801fa40 100644 --- a/demos/ordering-matters/README.md +++ b/demos/ordering-matters/README.md @@ -1,76 +1,92 @@ -# Ordering Matters Demo +# Demo 5: Ordering Matters -Demonstrates that applying identical rules in different orders produces measurably different emergent behavior in agent-based models. +This demo tests whether LLM rule learning depends on **trajectory row ordering**. -## Concept +The same underlying trajectory log is shown to the LLM in three views: +- `forward`: chronological order +- `reversed`: reverse chronological order +- `shuffled`: random permutation -Three groups of foraging agents each have the same three behavioral rules: +The LLM is asked to infer the behavioral rule that generated the trajectories. +`analysis.py` then measures how much inferred rules diverge across orderings. -| Rule | Description | -|------|-------------| -| **SENSE** | Detect nearest food within sensor range | -| **MOVE** | Navigate toward food, follow advice, or random walk | -| **SHARE** | Communicate food locations with nearby agents | +## Files -The groups differ **only** in execution order: +- `ordering-matters.nlogo`: NetLogo simulation + ordering/inference/export workflow +- `rule-inference-template.yaml`: prompt template used for rule inference +- `config.txt`: LLM provider settings +- `analysis.py`: rule-divergence analysis + JSON/plot output +- `tests/test_analysis.py`: validation for loader/scoring/report/plot logic +- `results/`: sample inference data, summaries, and plot outputs -| Group | Color | Order | Tendency | -|-------|-------|-------|----------| -| A | Red circles | sense → move → share | Reacts to own observations first | -| B | Blue squares | share → sense → move | Communicates before acting | -| C | Green triangles | move → share → sense | Acts impulsively, reflects later | +## Model Workflow -## Files +1. `setup` +- Initializes food and agents. +- Agents follow one hidden policy: seek food, avoid local crowding, preserve momentum. -| File | Purpose | -|------|---------| -| `ordering-matters.nlogo` | NetLogo model with interface and BehaviorSpace experiment | -| `rule-inference-template.yaml` | LLM prompt template for inferring rule orderings from data | -| `config` | LLM provider configuration (OpenAI, Anthropic, Gemini, Ollama) | -| `analysis.py` | Python script to analyze exported simulation data | -| `tests/test_analysis.py` | Pytest suite for the analysis script | +2. `go` / `go-forever` +- Runs simulation and logs trajectory rows: `tick, agent, x, y, heading, energy, food`. -## Quick Start +3. `infer-rules` +- Builds 3 trajectory presentations: forward/reversed/shuffled. +- Sends each to the same YAML template. +- Stores one inferred rule per ordering. -1. Edit `config` with your LLM provider credentials (or disable `use-llm?` in the model) -2. Open `ordering-matters.nlogo` in NetLogo 6.4+ -3. Click **setup** then **go-forever** -4. Watch the plot diverge as rule ordering drives different outcomes -5. Click **export-data** to save CSV, then analyze: +4. `export-data` +- Writes trajectories to `results/-trajectories.csv`. +- Writes inferred rules to `results/-inference.csv`. -```bash -python3 analysis.py ordering-matters-output.csv --plot results.png -``` +## Configure LLM -## Running Without LLM +Edit `config.txt` and choose one provider block. -The model works without any LLM provider. Turn off the `use-llm?` switch and agents will share raw coordinates instead of natural-language messages. This mode is useful for fast batch experiments via BehaviorSpace. +Default file includes examples for: +- OpenAI +- Anthropic +- Gemini +- Ollama + +If you disable `use-llm?` in NetLogo, inference uses a deterministic baseline text (useful for dry runs). ## Analysis -The `analysis.py` script reads exported CSV data and produces: +Run analysis on any exported inference CSV: + +```bash +cd demos/ordering-matters +python3 analysis.py results/sample-inference.csv \ + --json results/analysis-summary.json \ + --plot results/rule-similarity.png +``` + +### Metrics + +- `jaccard`: token overlap of normalized inferred rules +- `sequence`: string-order similarity (SequenceMatcher) +- `combined`: mean of jaccard + sequence +- `order_dependency_score`: `1 - average(combined_pairwise_similarity)` -- Per-group metrics (food collected, food rate, energy efficiency) -- Pairwise effect sizes between groups -- Summary report identifying the best-performing ordering -- Optional comparison plot (`--plot output.png`) +Interpretation: +- Higher `order_dependency_score` => stronger dependence on ordering +- `is_order_sensitive` is true when score exceeds threshold (`--threshold`, default `0.35`) -### Running Tests +## Tests ```bash cd demos/ordering-matters -python3 -m pytest tests/ -v +python3 -m pytest tests -q ``` -## Parameters - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `num-agents` | 18 | Total agents (divided into 3 equal groups) | -| `food-count` | 80 | Initial food patches | -| `sensor-range` | 4 | How far agents can detect food | -| `speed` | 1.0 | Distance agents move per tick | -| `comm-range` | 6 | Range for sharing information | -| `share-interval` | 5 | Ticks between communication attempts | -| `respawn-food?` | on | Whether food regenerates periodically | -| `respawn-interval` | 20 | Ticks between food respawn waves | +## Suggested Experiment Protocol + +1. Run 20+ simulation seeds. +2. Export one inference CSV per run. +3. Analyze each run via `analysis.py`. +4. Aggregate `order_dependency_score` distributions. +5. Compare across model families (e.g., GPT/Claude/Gemini/Ollama). + +## Notes + +- This demo measures **inference sensitivity**, not simulator sensitivity. +- Forward/reversed/shuffled use identical trajectory content; only row order changes.