diff --git a/docs/5vse.png b/docs/5vse.png index 68eefb8..1314cc4 100644 Binary files a/docs/5vse.png and b/docs/5vse.png differ diff --git a/docs/5vse_small.png b/docs/5vse_small.png index af89e3e..34a6a61 100644 Binary files a/docs/5vse_small.png and b/docs/5vse_small.png differ diff --git a/docs/irv-simulation.md b/docs/irv-simulation.md index ac88cfb..6ec3ded 100644 --- a/docs/irv-simulation.md +++ b/docs/irv-simulation.md @@ -23,3 +23,7 @@ The runner divides the work into ten fixed, seeded chunks and executes the chunk | 100% strategic | 60.50% | The embedded PNG charts on this site use these values. + +The static VSE charts show 95% normal-approximation confidence intervals for +the IRV/RCV points. The calculation is based on the election-level VSE values +from the same seeded run; the CSV output includes each interval's half-width. diff --git a/docs/vse.png b/docs/vse.png index 17b0a4f..7dd93d9 100644 Binary files a/docs/vse.png and b/docs/vse.png differ diff --git a/docs/vsestrat.png b/docs/vsestrat.png index ec61a29..0d133ea 100644 Binary files a/docs/vsestrat.png and b/docs/vsestrat.png differ diff --git a/scripts/recalculate_irv_pages.py b/scripts/recalculate_irv_pages.py index 7d0d5d5..36cd5dc 100644 --- a/scripts/recalculate_irv_pages.py +++ b/scripts/recalculate_irv_pages.py @@ -9,6 +9,7 @@ import argparse import csv +import math import os from collections import defaultdict from concurrent.futures import ProcessPoolExecutor @@ -30,8 +31,8 @@ def _recalculate_chunk(elections, seed): method = Irv() scenario_method = Schulze() media = fuzzyMediaFor() - summary = defaultdict(lambda: [0, 0.0]) - scenario_summary = defaultdict(lambda: defaultdict(lambda: [0, 0.0])) + summary = defaultdict(lambda: [0, 0.0, 0.0]) + scenario_summary = defaultdict(lambda: defaultdict(lambda: [0, 0.0, 0.0])) scenario_outcomes = defaultdict( lambda: {"attempts": 0, "successes": 0, "backfires": 0} ) @@ -51,9 +52,11 @@ def _recalculate_chunk(elections, seed): count_and_total = summary[row["chooser"]] count_and_total[0] += 1 count_and_total[1] += row["vse"] + count_and_total[2] += row["vse"] ** 2 scenario_count_and_total = scenario_summary[scenario][row["chooser"]] scenario_count_and_total[0] += 1 scenario_count_and_total[1] += row["vse"] + scenario_count_and_total[2] += row["vse"] ** 2 if row["chooser"] == "Oss.hon_strat.": one_sided_strategy["attempts"] += 1 scenario_outcomes[scenario]["attempts"] += 1 @@ -90,37 +93,49 @@ def recalculate(elections, seed, workers=DEFAULT_WORKERS): with ProcessPoolExecutor(max_workers=min(os.cpu_count() or 1, chunks)) as executor: chunks = list(executor.map(_recalculate_chunk, chunk_sizes, chunk_seeds)) - summary = defaultdict(lambda: [0, 0.0]) - scenario_summary = defaultdict(lambda: defaultdict(lambda: [0, 0.0])) + summary = defaultdict(lambda: [0, 0.0, 0.0]) + scenario_summary = defaultdict(lambda: defaultdict(lambda: [0, 0.0, 0.0])) one_sided_strategy = {"attempts": 0, "successes": 0, "backfires": 0} scenario_outcomes = defaultdict( lambda: {"attempts": 0, "successes": 0, "backfires": 0} ) for chunk_summary, chunk_outcomes, chunk_scenarios, chunk_scenario_outcomes in chunks: - for chooser, (count, total) in chunk_summary.items(): + for chooser, (count, total, total_squared) in chunk_summary.items(): summary[chooser][0] += count summary[chooser][1] += total + summary[chooser][2] += total_squared for key in one_sided_strategy: one_sided_strategy[key] += chunk_outcomes[key] for scenario, strategies in chunk_scenarios.items(): - for chooser, (count, total) in strategies.items(): + for chooser, (count, total, total_squared) in strategies.items(): scenario_summary[scenario][chooser][0] += count scenario_summary[scenario][chooser][1] += total + scenario_summary[scenario][chooser][2] += total_squared for scenario, outcomes in chunk_scenario_outcomes.items(): for key in outcomes: scenario_outcomes[scenario][key] += outcomes[key] + results = {chooser: total / count for chooser, (count, total, _sum_sq) in summary.items()} + confidence_intervals = {} + for chooser, (count, total, total_squared) in summary.items(): + if count < 2: + confidence_intervals[chooser] = 0.0 + continue + variance = max(0.0, (total_squared - total ** 2 / count) / (count - 1)) + confidence_intervals[chooser] = 1.96 * math.sqrt(variance / count) + return ( - {chooser: total / count for chooser, (count, total) in summary.items()}, + results, one_sided_strategy, { scenario: { chooser: total / count - for chooser, (count, total) in strategy_results.items() + for chooser, (count, total, _sum_sq) in strategy_results.items() } for scenario, strategy_results in scenario_summary.items() }, scenario_outcomes, + confidence_intervals, ) @@ -135,17 +150,19 @@ def main(): parser.add_argument("--output", type=Path) args = parser.parse_args() - results, one_sided_strategy, scenario_results, scenario_outcomes = recalculate( + results, one_sided_strategy, scenario_results, scenario_outcomes, confidence_intervals = recalculate( args.elections, args.seed, args.workers ) rows = [ - (chooser, args.elections, value, 100 * value) + (chooser, args.elections, value, confidence_intervals[chooser], 100 * value, + 100 * (value - confidence_intervals[chooser]), + 100 * (value + confidence_intervals[chooser])) for chooser, value in sorted(results.items()) ] if args.output: with args.output.open("w", newline="") as output: writer = csv.writer(output) - writer.writerow(["chooser", "elections", "mean_vse", "percent_vse"]) + writer.writerow(["chooser", "elections", "mean_vse", "ci95_half_width", "percent_vse", "ci95_low_percent", "ci95_high_percent"]) writer.writerows(rows) writer.writerow([]) writer.writerow(["strategy", "attempts", "success_rate", "backfire_rate"]) diff --git a/scripts/regenerate_pages_images.py b/scripts/regenerate_pages_images.py index 38dee0b..d0b0f40 100644 --- a/scripts/regenerate_pages_images.py +++ b/scripts/regenerate_pages_images.py @@ -144,7 +144,7 @@ def refresh_interactive_charts(results, outcomes, scenario_results, scenario_out replace_embedded_data(strategy_breakdown_path, strategy_breakdown) -def render_vse(data, output, size, selected=False): +def render_vse(data, output, size, intervals, selected=False): points = list(zip(data["x"], data["y"], data["col_var"], strict=False)) if selected: points = [point for point in points if point[1] in SMALL_METHODS] @@ -157,6 +157,16 @@ def render_vse(data, output, size, selected=False): positions = {label: index + 1 for index, label in enumerate(labels)} figure, axis = plt.subplots(figsize=(size[0] / 100, size[1] / 100), dpi=100) for value, method, strategy in points: + if "IRV/RCV" in method: + axis.errorbar( + value, + positions[method], + xerr=intervals[IRV_CHOOSERS[strategy]], + color=COLORS[strategy], + capsize=2, + linewidth=1, + zorder=2, + ) axis.scatter(value, positions[method], color=COLORS[strategy], s=34, zorder=3) axis.set_xlim(0.55 if selected else -0.2, 1.0) @@ -214,15 +224,15 @@ def main(): help="Deterministic simulation chunks (default: 10).", ) args = parser.parse_args() - results, outcomes, scenario_results, scenario_outcomes = recalculate( + results, outcomes, scenario_results, scenario_outcomes, intervals = recalculate( args.elections, args.seed, args.workers ) data = refreshed_vse_data(results) docs = ROOT / "docs" refresh_interactive_charts(results, outcomes, scenario_results, scenario_outcomes) - render_vse(data, docs / "vse.png", (970, 681)) - render_vse(data, docs / "5vse.png", (952, 567), selected=True) - render_vse(data, docs / "5vse_small.png", (656, 271), selected=True) + render_vse(data, docs / "vse.png", (970, 681), intervals) + render_vse(data, docs / "5vse.png", (952, 567), intervals, selected=True) + render_vse(data, docs / "5vse_small.png", (656, 271), intervals, selected=True) render_strategy(results, outcomes, docs / "vsestrat.png") diff --git a/tests/test_regressions.py b/tests/test_regressions.py index f155e6f..7b0b890 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -155,7 +155,7 @@ def test_csv_batch_can_stream_without_retaining_rows(tmp_path): def test_irv_recalculation_smoke(): - results, outcomes, scenarios, scenario_outcomes = recalculate( + results, outcomes, scenarios, scenario_outcomes, intervals = recalculate( elections=2, seed="test-irv", workers=1, @@ -163,6 +163,7 @@ def test_irv_recalculation_smoke(): assert "honBallot" in results assert outcomes["attempts"] == 2 + assert intervals["honBallot"] >= 0 assert sum(data["attempts"] for data in scenario_outcomes.values()) == 2 assert set(scenarios).issubset( {"cycle", "easy", "spoiler", "squeeze", "chicken", "other"}