From e2535d76444e4a64912297f43aaae0be0f4c38c3 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 11:24:04 +0800 Subject: [PATCH 01/10] {"schema":"decodex/commit/1","summary":"Add qmd and OpenViking strength-profile benchmark report","authority":"XY-899"} --- README.md | 14 +- .../memory_projects_manifest.json | 40 +- .../tests/real_world_job_benchmark.rs | 275 ++++++++++++- ...-11-competitor-strength-evidence-matrix.md | 9 +- ...on-direction-from-competitor-benchmarks.md | 11 +- ...6-06-11-elf-qmd-retrieval-debug-profile.md | 6 +- .../2026-06-11-measurement-coverage-audit.md | 62 +-- ...-qmd-openviking-strength-profile-report.md | 114 ++++++ docs/guide/benchmarking/index.md | 4 + ...md-openviking-strength-profile-report.json | 362 ++++++++++++++++++ ...-11-xy-897-competitor-strength-matrix.json | 6 +- 11 files changed, 830 insertions(+), 73 deletions(-) create mode 100644 docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md create mode 100644 docs/research/2026-06-11-qmd-openviking-strength-profile-report.json diff --git a/README.md b/README.md index b4032dde..95285c66 100644 --- a/README.md +++ b/README.md @@ -153,12 +153,15 @@ provider-backed ELF evidence was required. jobs across 11 suites, 36 pass, 0 incomplete, 2 blocked, 0 wrong-result, 0 not-encoded, and 0 unsupported-claim results. The remaining non-pass jobs are production-ops operator boundaries, not hidden benchmark wins. -- Full-suite live real-world adapter sweep after XY-880: ELF and qmd now emit +- Full-suite live real-world adapter sweep after XY-899: ELF and qmd emit Docker-isolated `live_real_world` records for all 38 encoded jobs across 11 suites through `cargo make real-world-memory-live-adapters`. Both keep the original targeted `work_resume`, `retrieval`, and `project_decisions` slice passing, but the - full sweep is not a full-suite pass: each adapter reports 18 pass, 5 wrong_result, - 1 incomplete, 2 blocked, and 12 not_encoded jobs. + full sweep is not a full-suite pass. The fresh ELF sweep reports 18 pass, + 5 wrong_result, 2 blocked, and 13 not_encoded jobs. The fresh qmd sweep reports + 17 pass, 6 wrong_result, 2 blocked, and 13 not_encoded jobs. The difference is the + delete/TTL tombstone case; qmd remains the local retrieval-debug UX reference, and + no broad ELF-over-qmd claim is allowed. - Expanded adapter-pack coverage after XY-834: the real-world external adapter manifest now includes `research_gate` records for RAGFlow, LightRAG, GraphRAG, Graphiti/Zep, Letta, LangGraph, nanograph, llm-wiki, gbrain, graphify, and deeper @@ -183,6 +186,7 @@ Detailed evidence and interpretation: - [Real-World Comparison Report - June 10, 2026](docs/guide/benchmarking/2026-06-10-real-world-comparison-report.md) - [Live Real-World Adapter Sweep Report - June 10, 2026](docs/guide/benchmarking/2026-06-10-live-real-world-sweep-report.md) - [Post-Adapter Production Adoption Refresh - June 10, 2026](docs/guide/benchmarking/2026-06-10-production-adoption-refresh.md) +- [qmd and OpenViking Strength-Profile Report - June 11, 2026](docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md) - [Live Baseline Benchmark Runbook](docs/guide/benchmarking/live_baseline_benchmark.md) - [Single-User Production Runbook](docs/guide/single_user_production.md) - Benchmark contract: @@ -263,8 +267,8 @@ Detailed comparison, mechanism-level analysis, and source map: - [Real-World Benchmark Dimension Research Run](docs/research/2026-06-09-xy-841-external-memory-benchmark-dimensions.json) - [RAG/Graph Adapter Feasibility Research Run](docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json) -Latest real-world benchmark report: June 10, 2026. Latest external research refresh: -June 10, 2026. +Latest real-world benchmark report: June 11, 2026. Latest external research refresh: +June 11, 2026. ## Documentation diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 152b1f15..33e15d4b 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -1,6 +1,6 @@ { "schema": "elf.real_world_external_adapter_manifest/v1", - "manifest_id": "real-world-memory-project-adapters-2026-06-10", + "manifest_id": "real-world-memory-project-adapters-2026-06-11", "docker_isolation": { "default": true, "compose_file": "docker-compose.baseline.yml", @@ -146,13 +146,13 @@ }, "run": { "status": "wrong_result", - "evidence": "ELF materializes 38 real_world_job adapter_response objects through ElfService, worker indexing, and search_raw before scoring; the full sweep includes typed wrong_result, incomplete, blocked, and not_encoded records.", + "evidence": "ELF materializes 38 real_world_job adapter_response objects through ElfService, worker indexing, and search_raw before scoring; the fresh full sweep includes typed wrong_result, blocked, and not_encoded records.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/elf-report.json" }, "result": { "status": "wrong_result", - "evidence": "The full live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 1 incomplete, 2 blocked, and 12 not_encoded. This is not a full-suite live pass.", + "evidence": "The fresh ELF live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 2 blocked, and 13 not_encoded. This is not a full-suite live pass.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/elf-report.md" }, @@ -180,7 +180,7 @@ { "capability": "full_suite_live_pass", "status": "wrong_result", - "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, incomplete, blocked, and not_encoded outcomes." + "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, blocked, and not_encoded outcomes." }, { "capability": "typed_failure_reporting", @@ -236,8 +236,8 @@ }, { "suite_id": "production_ops", - "status": "incomplete", - "evidence": "The live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked and the cold-start dependency fixture remains incomplete." + "status": "blocked", + "evidence": "The live adapter sweep preserves encoded production-ops operator-boundary jobs as blocked; backup/restore, private corpus, provider credential, and backfill operations are not claimed as live adapter passes." }, { "suite_id": "personalization", @@ -264,7 +264,7 @@ ], "notes": [ "This Docker-isolated live real_world_job record now covers the full encoded fixture corpus, not only the original three-suite representative slice.", - "The record is a full-suite sweep, not a full-suite pass; wrong_result, incomplete, blocked, and not_encoded states remain visible.", + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible.", "This record does not prove private-corpus production quality or provider-backed production operations." ] }, @@ -359,13 +359,13 @@ }, "run": { "status": "wrong_result", - "evidence": "qmd materializes 38 real_world_job adapter_response objects through collection add, update, embed, and query --json before scoring; the full sweep includes typed wrong_result, incomplete, blocked, and not_encoded records.", + "evidence": "qmd materializes 38 real_world_job adapter_response objects through collection add, update, embed, and query --json before scoring; the fresh full sweep includes typed wrong_result, blocked, and not_encoded records.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/qmd-report.json" }, "result": { "status": "wrong_result", - "evidence": "The full qmd live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 1 incomplete, 2 blocked, and 12 not_encoded. This is not a full-suite live pass.", + "evidence": "The fresh qmd live sweep scores 38 jobs across all 11 encoded suites: 17 pass, 6 wrong_result, 2 blocked, and 13 not_encoded. This is not a full-suite live pass.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/qmd-report.md" }, @@ -393,7 +393,7 @@ { "capability": "full_suite_live_pass", "status": "wrong_result", - "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, incomplete, blocked, and not_encoded outcomes." + "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, blocked, and not_encoded outcomes." }, { "capability": "typed_failure_reporting", @@ -425,7 +425,7 @@ { "suite_id": "memory_evolution", "status": "wrong_result", - "evidence": "qmd passed the delete/TTL case but failed five current-versus-historical conflict jobs because retrieval-backed answers did not provide the required historical conflict evidence links." + "evidence": "qmd failed all six memory-evolution jobs in the fresh June 11 diagnostic, including the delete/TTL tombstone job where qmd retrieved only the current plan and missed the tombstone evidence." }, { "suite_id": "consolidation", @@ -449,8 +449,8 @@ }, { "suite_id": "production_ops", - "status": "incomplete", - "evidence": "The qmd live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked and the cold-start dependency fixture remains incomplete." + "status": "blocked", + "evidence": "The qmd live adapter sweep preserves the encoded production-ops operator-boundary jobs as blocked; no qmd production-ops pass claim is allowed from this adapter." }, { "suite_id": "personalization", @@ -477,7 +477,7 @@ ], "notes": [ "This qmd record is real-world job evidence and must not be conflated with the same-corpus qmd_live_baseline record.", - "The record is a full-suite sweep, not a full-suite pass; wrong_result, incomplete, blocked, and not_encoded states remain visible.", + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible.", "This record does not prove broad RAG/graph adapter parity or private-corpus production quality." ] }, @@ -917,11 +917,12 @@ }, "run": { "status": "not_encoded", - "evidence": "No expanded qmd stress or real_world_job deep-profile artifact is checked in for this adapter-pack gate." + "evidence": "The XY-899 strength-profile report is checked in, but no new live qmd deep-profile adapter artifact is claimed from it." }, "result": { "status": "not_encoded", - "evidence": "qmd deep retrieval-debug evidence remains a planned profile, not a new pass claim." + "evidence": "The XY-899 report records qmd scenario-level retrieval/debug/replay outcomes and wrong-result diagnosis taxonomy, while expansion/fusion/rerank scoring remains not_encoded.", + "artifact": "docs/research/2026-06-11-qmd-openviking-strength-profile-report.json" }, "capabilities": [ { @@ -988,7 +989,7 @@ { "adapter_id": "openviking_deep_profile_gate", "project": "OpenViking", - "adapter_kind": "docker_local_embed_deep_profile_gate", + "adapter_kind": "docker_local_embed_context_trajectory_gate", "evidence_class": "research_gate", "docker_default": true, "host_global_installs_required": false, @@ -1001,11 +1002,12 @@ }, "run": { "status": "not_encoded", - "evidence": "The adapter cannot fairly exercise hierarchical trajectory behavior until same-corpus add_resource/find returns evidence-bearing results." + "evidence": "The XY-899 strength-profile report records staged retrieval, hierarchy selection, recursive/context expansion, and missed-term evidence as typed not_tested or wrong_result states; no new live trajectory adapter artifact is claimed." }, "result": { "status": "not_encoded", - "evidence": "No OpenViking deep context-trajectory result is claimed from the current wrong-result smoke run." + "evidence": "No OpenViking deep context-trajectory result is claimed from the current wrong-result smoke run; the XY-899 report preserves the trajectory surfaces as not_tested.", + "artifact": "docs/research/2026-06-11-qmd-openviking-strength-profile-report.json" }, "capabilities": [ { diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index b8f14a81..154f8ec3 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -60,6 +60,54 @@ fn production_ops_fixture_dir() -> PathBuf { real_world_memory_fixture_dir().join("production_ops") } +fn workspace_root() -> Result { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let root = manifest_dir + .parent() + .and_then(Path::parent) + .ok_or_else(|| eyre::eyre!("could not resolve workspace root"))?; + + Ok(root.to_path_buf()) +} + +fn strength_profile_report_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-qmd-openviking-strength-profile-report.json")) +} + +fn strength_profile_markdown_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-qmd-openviking-strength-profile-report.md")) +} + +fn measurement_coverage_audit_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-measurement-coverage-audit.md")) +} + +fn competitor_strength_matrix_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-competitor-strength-evidence-matrix.md")) +} + +fn external_adapter_manifest_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join("real_world_external_adapters") + .join("memory_projects_manifest.json") +} + fn run_json_report_from(fixtures: PathBuf) -> Result { let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) .arg("run") @@ -191,7 +239,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { ); assert_eq!( report.pointer("/external_adapters/manifest_id").and_then(Value::as_str), - Some("real-world-memory-project-adapters-2026-06-10") + Some("real-world-memory-project-adapters-2026-06-11") ); assert_eq!( report.pointer("/external_adapters/docker_isolation/default").and_then(Value::as_bool), @@ -281,7 +329,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/suite_status_counts/blocked") .and_then(Value::as_u64), - Some(11) + Some(13) ); } @@ -302,6 +350,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { let graphiti_zep = find_by_field(adapters, "/adapter_id", "graphiti_zep_research_gate")?; let graphify = find_by_field(adapters, "/adapter_id", "graphify_research_gate")?; let qmd_deep = find_by_field(adapters, "/adapter_id", "qmd_deep_profile_gate")?; + let openviking_deep = find_by_field(adapters, "/adapter_id", "openviking_deep_profile_gate")?; assert_eq!(elf.pointer("/evidence_class").and_then(Value::as_str), Some("fixture_backed")); assert_eq!(elf.pointer("/overall_status").and_then(Value::as_str), Some("blocked")); @@ -311,7 +360,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { ); assert_eq!(elf_live.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); - assert_live_sweep_record(elf_live)?; + assert_live_sweep_record(elf_live, "blocked")?; assert_eq!(qmd.pointer("/overall_status").and_then(Value::as_str), Some("pass")); assert_eq!(qmd.pointer("/suites/0/status").and_then(Value::as_str), Some("not_encoded")); @@ -321,7 +370,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { ); assert_eq!(qmd_live.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); - assert_live_sweep_record(qmd_live)?; + assert_live_sweep_record(qmd_live, "blocked")?; assert_eq!( agentmemory.pointer("/capabilities/1/status").and_then(Value::as_str), @@ -379,6 +428,18 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { qmd_deep.pointer("/capabilities/2/status").and_then(Value::as_str), Some("unsupported") ); + assert_eq!( + qmd_deep.pointer("/result/artifact").and_then(Value::as_str), + Some("docs/research/2026-06-11-qmd-openviking-strength-profile-report.json") + ); + assert_eq!( + openviking_deep.pointer("/adapter_kind").and_then(Value::as_str), + Some("docker_local_embed_context_trajectory_gate") + ); + assert_eq!( + openviking_deep.pointer("/result/artifact").and_then(Value::as_str), + Some("docs/research/2026-06-11-qmd-openviking-strength-profile-report.json") + ); Ok(()) } @@ -454,7 +515,7 @@ fn assert_graphify_adapter(adapter: &Value) { ); } -fn assert_live_sweep_record(adapter: &Value) -> Result<()> { +fn assert_live_sweep_record(adapter: &Value, production_ops_status: &str) -> Result<()> { let suites = array_at(adapter, "/suites")?; let capabilities = array_at(adapter, "/capabilities")?; let targeted = find_by_field(capabilities, "/capability", "targeted_live_pass")?; @@ -468,7 +529,10 @@ fn assert_live_sweep_record(adapter: &Value) -> Result<()> { assert_eq!(full_pass.pointer("/status").and_then(Value::as_str), Some("wrong_result")); assert_eq!(work_resume.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(memory_evolution.pointer("/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!(production_ops.pointer("/status").and_then(Value::as_str), Some("incomplete")); + assert_eq!( + production_ops.pointer("/status").and_then(Value::as_str), + Some(production_ops_status) + ); assert_eq!(consolidation.pointer("/status").and_then(Value::as_str), Some("not_encoded")); Ok(()) @@ -757,6 +821,205 @@ fn project_decisions_fixtures_report_decision_policy_cases() -> Result<()> { Ok(()) } +#[test] +fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result<()> { + let report = + serde_json::from_str::(&fs::read_to_string(strength_profile_report_path()?)?)?; + let markdown = fs::read_to_string(strength_profile_markdown_path()?)?; + + assert_strength_profile_summary(&report); + assert_qmd_strength_profile(&report)?; + assert_qmd_wrong_result_diagnosis(&report)?; + assert_openviking_strength_profile(&report)?; + assert_strength_profile_markdown_boundaries(&markdown); + + Ok(()) +} + +#[test] +fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { + let measurement_audit = fs::read_to_string(measurement_coverage_audit_path()?)?; + let competitor_matrix = fs::read_to_string(competitor_strength_matrix_path()?)?; + let external_manifest = fs::read_to_string(external_adapter_manifest_path())?; + + assert!( + measurement_audit.contains( + "| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | `wrong_result:6` |" + ) + ); + assert!( + measurement_audit + .contains("qmd live fails 6/6 jobs after missing the delete/TTL tombstone evidence") + ); + assert!( + competitor_matrix + .contains("broader live suites remain `wrong_result`, `blocked`, or `not_encoded`") + ); + assert!(external_manifest.contains( + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible." + )); + + for stale_phrase in [ + "same live sweep shape as ELF", + "ELF and qmd live fail 5/6 jobs", + "both systems currently fail 5/6 live memory-evolution jobs", + "wrong_result, incomplete, blocked, and not_encoded states remain visible", + "broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`", + ] { + assert!(!measurement_audit.contains(stale_phrase)); + assert!(!competitor_matrix.contains(stale_phrase)); + assert!(!external_manifest.contains(stale_phrase)); + } + + Ok(()) +} + +fn assert_strength_profile_summary(report: &Value) { + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.competitor_strength_profile_report/v1") + ); + assert_eq!( + report.pointer("/summary/qmd/retrieval_quality").and_then(Value::as_str), + Some("tie") + ); + assert_eq!( + report.pointer("/summary/qmd/debug_replay_ergonomics").and_then(Value::as_str), + Some("elf_loss") + ); + assert_eq!( + report.pointer("/summary/openviking/overall_against_strengths").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!( + report + .pointer("/qmd_strength_profile/win_tie_loss_summary/elf_win") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report.pointer("/qmd_strength_profile/win_tie_loss_summary/tie").and_then(Value::as_u64), + Some(3) + ); + assert_eq!( + report + .pointer("/qmd_strength_profile/win_tie_loss_summary/elf_loss") + .and_then(Value::as_u64), + Some(2) + ); + assert_eq!( + report + .pointer("/qmd_strength_profile/win_tie_loss_summary/not_tested") + .and_then(Value::as_u64), + Some(3) + ); + assert_eq!( + report + .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/not_tested") + .and_then(Value::as_u64), + Some(6) + ); +} + +fn assert_qmd_strength_profile(report: &Value) -> Result<()> { + let qmd_scenarios = array_at(report, "/qmd_strength_profile/scenario_outcomes")?; + let local_transparency = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-local-query-transparency")?; + let retrieval = find_by_field(qmd_scenarios, "/scenario_id", "qmd-retrieval-quality")?; + let rerank_controls = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-expansion-fusion-rerank-controls")?; + let wrong_result = find_by_field(qmd_scenarios, "/scenario_id", "qmd-wrong-result-diagnosis")?; + + assert_eq!(qmd_scenarios.len(), 8); + assert_eq!(retrieval.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); + assert_eq!( + local_transparency.pointer("/elf_outcome").and_then(Value::as_str), + Some("elf_loss") + ); + assert_eq!( + rerank_controls.pointer("/result_type").and_then(Value::as_str), + Some("not_encoded") + ); + assert_eq!( + wrong_result.pointer("/evidence_class").and_then(Value::as_str), + Some("research_gate") + ); + assert_eq!(wrong_result.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + + Ok(()) +} + +fn assert_qmd_wrong_result_diagnosis(report: &Value) -> Result<()> { + let taxonomy = array_at(report, "/qmd_strength_profile/wrong_result_diagnosis/taxonomy")?; + let absent = find_by_field(taxonomy, "/class", "evidence_absent")?; + let dropped = find_by_field(taxonomy, "/class", "retrieved_but_dropped")?; + let narrated = find_by_field(taxonomy, "/class", "selected_but_not_narrated")?; + let lifecycle = find_by_field(taxonomy, "/class", "contradicted_by_lifecycle_evidence")?; + + assert_eq!(absent.pointer("/coverage").and_then(Value::as_str), Some("observed")); + assert_eq!( + dropped.pointer("/coverage").and_then(Value::as_str), + Some("not_observed_candidate_trace_missing") + ); + assert_eq!(narrated.pointer("/coverage").and_then(Value::as_str), Some("observed")); + assert_eq!(lifecycle.pointer("/coverage").and_then(Value::as_str), Some("observed")); + + let qmd_diagnosis_jobs = array_at(report, "/qmd_strength_profile/wrong_result_diagnosis/jobs")?; + let delete_job = + find_by_field(qmd_diagnosis_jobs, "/job_id", "memory-evolution-delete-ttl-001")?; + + assert_eq!(qmd_diagnosis_jobs.len(), 6); + assert_eq!(delete_job.pointer("/qmd_status").and_then(Value::as_str), Some("wrong_result")); + assert!(array_contains_str(delete_job, "/missing_evidence", "delete-tombstone")?); + assert!( + delete_job + .pointer("/diagnosis") + .and_then(Value::as_str) + .is_some_and(|diagnosis| diagnosis.contains("typed wrong_result")) + ); + + Ok(()) +} + +fn assert_openviking_strength_profile(report: &Value) -> Result<()> { + let openviking_scenarios = + array_at(report, "/openviking_context_trajectory_profile/scenario_outcomes")?; + let trajectory = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-staged-retrieval-trajectory", + )?; + let precondition = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-evidence-bearing-retrieval-precondition", + )?; + + assert_eq!(openviking_scenarios.len(), 6); + assert_eq!( + trajectory.pointer("/evidence_class").and_then(Value::as_str), + Some("research_gate") + ); + assert_eq!(trajectory.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(precondition.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + precondition.pointer("/typed_blocker").and_then(Value::as_str), + Some("output_missed_expected_terms") + ); + + Ok(()) +} + +fn assert_strength_profile_markdown_boundaries(markdown: &str) { + assert!( + markdown.contains( + "| Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` |" + ) + ); + assert!(markdown.contains("no pass evidence is claimed")); + assert!(markdown.contains("typed `wrong_result` state")); +} + #[test] fn generated_json_report_renders_markdown() -> Result<()> { let report = run_json_report()?; diff --git a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md index 1802eaf5..84f25f53 100644 --- a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md +++ b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md @@ -26,8 +26,9 @@ is encoded and run at a comparable evidence class. Current boundary: - ELF and qmd have full-suite `live_real_world` sweeps, but neither has a full-suite - live pass. Each sweep produced 38 jobs with 18 pass, 5 wrong_result, 1 incomplete, - 2 blocked, and 12 not_encoded. + live pass. The fresh ELF sweep produced 38 jobs with 18 pass, 5 wrong_result, + 2 blocked, and 13 not_encoded; the fresh qmd sweep produced 38 jobs with 17 pass, + 6 wrong_result, 2 blocked, and 13 not_encoded. - ELF fixture evidence is strong: `cargo make real-world-memory` reports 38 jobs across 11 suites with 36 pass and 2 blocked production-ops operator boundaries. That proves the fixture contract, not live-service parity. @@ -71,7 +72,7 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Project | Strongest user-facing scenario | Current evidence | Measured status and proof | Unsupported or blocked status | Required benchmark before ELF claim | Borrow if stronger | | --- | --- | --- | --- | --- | --- | --- | -| ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | +| ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `blocked`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | | qmd | Local retrieval-debug workflow with transparent CLI indexing, querying, expansion, fusion, and rerank ergonomics. | `live_real_world`; supporting `live_baseline_only` and `research_gate`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/qmd-report.md`; targeted retrieval suites pass. | `not_encoded`: deep profile and non-retrieval live behavior are not encoded; memory_evolution is `wrong_result`. | qmd deep retrieval/debug profile plus full-suite live replay with trace-level diagnostics. | Weighted fusion, rerank explanation, local debug knobs, and command-line replay. | | agentmemory | Coding-agent continuity, MCP/REST packaging, viewer workflow, and durable cross-agent memory lifecycle. | `live_baseline_only`. | `lifecycle_fail`: `ELF_BASELINE_PROJECTS=agentmemory cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `blocked`: durable cold-start and real-world adapter coverage are missing. | Durable local adapter with update, delete, cold-start reload, work_resume, capture/write-policy, and lifecycle-staleness jobs. | Cross-agent hooks, packaging, continuity scenarios, and viewer affordances. | | mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=mem0 cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: OpenMemory UI, hosted claims, and real-world personalization coverage are not encoded. | Fix local same-corpus result, then encode memory_evolution, personalization, UI readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | @@ -102,7 +103,7 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Knowledge pages | Fixture knowledge_compilation passes; live knowledge_compilation is `not_encoded`. | llm-wiki, gbrain, GraphRAG, graphify. | llm-wiki and gbrain are `research_gate` `not_encoded` or `blocked`; GraphRAG and graphify are `blocked`. | Encode live derived-page rebuild/lint scoring and run contained knowledge/RAG adapters only after setup proof. | | Operator debugging | Fixture operator_debugging_ux passes; live operator_debugging_ux is `not_encoded`. | qmd, claude-mem, OpenMemory. | qmd has debug strengths but operator_debugging_ux is `not_encoded`; claude-mem and OpenMemory UX are `not_encoded`. | Score trace hydration, stage attribution, raw-SQL avoidance, and repair-action clarity through live artifacts. | | Capture/write policy | Fixture capture_integration passes; live capture_integration is `not_encoded`. | agentmemory, claude-mem. | agentmemory capture is `blocked`; claude-mem capture is `not_encoded`. | Run live capture/write-policy jobs proving redaction, exclusion, evidence binding, and no secret leakage. | -| Production ops | Fixture production_ops has 4 pass and 2 blocked; live production_ops is `incomplete`; production adoption has provider/backfill/restore evidence. | ELF production gate, qmd, RAG/RAGFlow resource gates. | qmd live production_ops is `incomplete`; RAG/resource gates are `research_gate` `blocked`. | Rerun private-corpus and credentialed gates only when operator-owned manifest and credentials exist. | +| Production ops | Fixture production_ops has 4 pass and 2 blocked; live production_ops is `blocked`; production adoption has provider/backfill/restore evidence. | ELF production gate, qmd, RAG/RAGFlow resource gates. | qmd live production_ops is `blocked`; RAG/resource gates are `research_gate` `blocked`. | Rerun private-corpus and credentialed gates only when operator-owned manifest and credentials exist. | | Personalization | Fixture and live personalization pass. | mem0/OpenMemory, Letta. | mem0/OpenMemory and Letta personalization are `not_encoded`. | Encode scoped preference readback for mem0/OpenMemory and Letta before personalization superiority claims. | | Context trajectory | ELF has trace direction but no comparable staged trajectory scenario. | OpenViking. | OpenViking setup is pinned, same-corpus retrieval is `wrong_result`, and hierarchy trajectory is `not_encoded`. | Make OpenViking evidence-bearing retrieval pass, then score staged context trajectory outputs. | | Core-vs-archival memory | ELF core-block semantics exist in the service contract, but comparative benchmark coverage is not encoded here. | Letta. | Letta is `research_gate` `not_encoded` until contained export proof exists. | Add ELF core-block versus archival-search jobs; compare Letta only after contained export proof. | diff --git a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md index d581b76c..b74dbaf1 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md +++ b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md @@ -66,17 +66,18 @@ sweeps for ELF and qmd: | Adapter | Jobs | Pass | Wrong result | Incomplete | Blocked | Not encoded | Mean score | Evidence recall | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `1` | `2` | `12` | `0.514` | `41/75` | -| qmd live CLI adapter | `38` | `18` | `5` | `1` | `2` | `12` | `0.512` | `41/75` | +| ELF live service adapter | `38` | `18` | `5` | `0` | `2` | `13` | `0.525` | `41/77` | +| qmd live CLI adapter | `38` | `17` | `6` | `0` | `2` | `13` | `0.486` | `38/77` | Interpretation: -- This is a tie for the currently encoded live real-world sweep. +- This is not a full-suite live pass for either system. ELF is one pass ahead in the + fresh aggregate because qmd misses the delete/TTL tombstone job. - Both pass `trust_source_of_truth`, `work_resume`, `project_decisions`, `retrieval`, and `personalization`. -- Both fail `memory_evolution` live conflict evidence with `wrong_result`. +- Both fail most `memory_evolution` live conflict evidence with `wrong_result`. - Both leave consolidation, knowledge compilation, operator debugging, capture - integration, and parts of production operations as `not_encoded` or incomplete. + integration, and parts of production operations as `not_encoded` or blocked. ### Production Evidence diff --git a/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md b/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md index 6e3af93e..8054b3fe 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md +++ b/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md @@ -67,11 +67,13 @@ Full live sweep context remains a non-pass for both systems: | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `5.823 ms` | -| qmd live CLI adapter | `38` | `18` | `5` | `2` | `13` | `0.512` | `705.877 ms` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `691.163 ms` | Do not overread the latency row. The ELF adapter is a service-runtime path and the qmd adapter is a CLI materialization path; the row is useful as observed harness evidence, -not as an apples-to-apples product latency benchmark. +not as an apples-to-apples product latency benchmark. The aggregate pass-count +difference comes from the memory-evolution delete/TTL tombstone job; it does not erase +qmd's local retrieval-debug ergonomics advantage. ## Stress Baseline diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index 862395b4..4c1a9657 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -23,9 +23,9 @@ What is proven today: - ELF has a strong fixture-backed real-world benchmark contract: 38 jobs, 36 pass, 2 blocked operator boundaries, and no wrong results in the fixture aggregate. -- ELF and qmd have comparable full-suite live real-world sweeps. They are effectively - tied on pass/fail shape: each has 38 jobs, 18 pass, 5 wrong_result, 2 blocked, and - 13 not_encoded. +- ELF and qmd have comparable full-suite live real-world sweeps, but neither has a + full-suite live pass. ELF is one pass ahead in the fresh aggregate because qmd + misses the memory-evolution delete/TTL tombstone job. - ELF is ahead on production-operation evidence among tracked systems because it has checked-in provider synthetic, stress, backfill, backup/restore, and Qdrant rebuild evidence. @@ -83,33 +83,36 @@ live adapter or competitor runtime can complete those jobs. | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | Evidence recall | Evidence coverage | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `5.100 ms` | `41/77` | `48/84` | -| qmd live CLI adapter | `38` | `18` | `5` | `2` | `13` | `0.512` | `719.758 ms` | `41/77` | `48/84` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `691.163 ms` | `38/77` | `45/84` | -This supports a narrow tie on the currently encoded live real-world suite shape. It -does not support a broad ELF-over-qmd claim because qmd remains the stronger -retrieval-debug UX reference and its deep profile is still not encoded. +This supports a narrow ELF lead only on the fresh aggregate count and only because of +the delete/TTL tombstone case. It does not support a broad ELF-over-qmd claim because +qmd remains the stronger retrieval-debug UX reference and its deep profile is still +not encoded. ### Live Suite Breakdown -ELF and qmd had the same suite status shape: - -| Suite | Jobs | Status breakdown | -| --- | ---: | --- | -| `trust_source_of_truth` | `1` | `pass:1` | -| `work_resume` | `5` | `pass:5` | -| `retrieval` | `5` | `pass:5` | -| `project_decisions` | `5` | `pass:5` | -| `personalization` | `1` | `pass:1` | -| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | -| `capture_integration` | `2` | `not_encoded:2` | -| `consolidation` | `4` | `not_encoded:4` | -| `knowledge_compilation` | `2` | `not_encoded:2` | -| `operator_debugging_ux` | `1` | `not_encoded:1` | -| `production_ops` | `6` | `blocked:2`, `not_encoded:4` | - -The five live wrong results are all memory-evolution jobs. The live adapters retrieve -current evidence but do not yet provide the required historical conflict evidence -links for current-vs-historical reasoning. +ELF and qmd have the same suite status shape for most encoded suites, but the fresh +memory-evolution diagnostic splits them on the delete/TTL tombstone job: + +| Suite | Jobs | ELF status breakdown | qmd status breakdown | +| --- | ---: | --- | --- | +| `trust_source_of_truth` | `1` | `pass:1` | `pass:1` | +| `work_resume` | `5` | `pass:5` | `pass:5` | +| `retrieval` | `5` | `pass:5` | `pass:5` | +| `project_decisions` | `5` | `pass:5` | `pass:5` | +| `personalization` | `1` | `pass:1` | `pass:1` | +| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | `wrong_result:6` | +| `capture_integration` | `2` | `not_encoded:2` | `not_encoded:2` | +| `consolidation` | `4` | `not_encoded:4` | `not_encoded:4` | +| `knowledge_compilation` | `2` | `not_encoded:2` | `not_encoded:2` | +| `operator_debugging_ux` | `1` | `not_encoded:1` | `not_encoded:1` | +| `production_ops` | `6` | `blocked:2`, `not_encoded:4` | `blocked:2`, `not_encoded:4` | + +The ELF live wrong results are five memory-evolution jobs. qmd has those same conflict +evidence failures plus the delete/TTL tombstone miss. The live adapters retrieve +current evidence in several cases but do not yet provide the required historical +conflict evidence links for current-vs-historical reasoning. ## External Adapter Ledger @@ -140,7 +143,7 @@ Interpret the unique manifest project list as the project coverage count. | Project | Best current evidence | Current measured state | Strongest unproven scenario | Next measurement before claim | | --- | --- | --- | --- | --- | | ELF | `fixture_backed` plus `live_real_world` | Fixture aggregate passes except 2 blocked operator boundaries; live full sweep is `wrong_result`. | Full live memory evolution, live consolidation, live knowledge pages, live capture, live production ops. | Memory-evolution diagnostic report, then live operator/capture/consolidation reports. | -| qmd | `live_real_world` plus `live_baseline_only` | Same live sweep shape as ELF; same-corpus baseline passes. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | +| qmd | `live_real_world` plus `live_baseline_only` | Fresh full sweep is one pass behind ELF because qmd misses the delete/TTL tombstone job; same-corpus baseline passes. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | | agentmemory | `live_baseline_only` | `lifecycle_fail`. | Durable coding-agent continuity and capture hooks. | Durable lifecycle and work-resume/capture adapter report. | | mem0/OpenMemory | `live_baseline_only` | `wrong_result`. | Entity history, lifecycle UI, OpenMemory inspection. | Same-corpus repair first, then entity-history and UI-readback report. | | memsearch | `live_baseline_only` | `wrong_result`; source-of-truth is `incomplete`. | Markdown canonical store and local reindex clarity. | Reindex/update/delete/reload plus source-of-truth report. | @@ -165,7 +168,7 @@ Interpret the unique manifest project list as the project coverage count. | Work resume | ELF and qmd live pass. | ELF is credible on encoded work resume. | agentmemory, claude-mem, and OpenViking comparable continuity adapters. | | Project decisions | ELF and qmd live pass. | ELF is credible on encoded project-decision recovery. | Letta core/archival decision memory comparison. | | Source of truth | ELF and qmd live pass; ELF has stronger production restore/rebuild evidence. | ELF has strongest measured source-of-truth discipline. | memsearch source-of-truth reindex/reload evidence. | -| Memory evolution | ELF and qmd live fail 5/6 jobs; fixture aggregate passes. | No live superiority claim. | Historical conflict evidence links and Graphiti/Zep temporal comparison. | +| Memory evolution | ELF live fails 5/6 jobs; qmd live fails 6/6 jobs after missing the delete/TTL tombstone evidence; fixture aggregate passes. | No broad live superiority claim. | Historical conflict evidence links and Graphiti/Zep temporal comparison. | | Consolidation | Fixture aggregate passes; live adapters are not encoded. | Fixture-only claim. | Live proposal generation with lineage, confidence, and review-action audit. | | Knowledge pages | Fixture aggregate passes; live adapters are not encoded. | Fixture-only claim. | Live page rebuild/lint plus llm-wiki, gbrain, GraphRAG, and graphify comparisons. | | Operator debugging | Fixture aggregate passes; live adapters are not encoded. | Fixture-only claim. | Trace hydration, stage attribution, dropped-candidate, and repair-action scoring. | @@ -187,7 +190,8 @@ Order these by decision value, not implementation convenience: rerank, dropped candidates, and command-line replay. 2. ELF/qmd live memory-evolution diagnostic - - Why: both systems currently fail 5/6 live memory-evolution jobs. + - Why: ELF currently fails 5/6 live memory-evolution jobs and qmd fails 6/6, + including the delete/TTL tombstone case. - Output: per-job evidence-link failure analysis for current-vs-historical facts, supersession, and relation temporal validity. diff --git a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md new file mode 100644 index 00000000..b0375c15 --- /dev/null +++ b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md @@ -0,0 +1,114 @@ +# qmd and OpenViking Strength-Profile Report - June 11, 2026 + +Goal: Compare ELF against qmd and OpenViking on their actual strengths without +turning broad live-sweep or smoke results into unsupported win claims. +Read this when: You need the XY-899 scenario-level qmd retrieval-debug and +OpenViking context-trajectory benchmark/report outcome. +Inputs: The June 11 retrieval-debug, memory-evolution, and temporal-history reports, +the real-world benchmark spec, the external adapter manifest, and +`scripts/real-world-live-adapters.sh`. +Outputs: Scenario-level win/tie/loss/not-tested judgments, qmd wrong-result +diagnosis taxonomy, OpenViking typed trajectory blockers, and claim boundaries. + +Machine-readable companion: +`docs/research/2026-06-11-qmd-openviking-strength-profile-report.json`. + +## Executive Judgment + +ELF does not have a broad win against either qmd or OpenViking on their strengths. + +The measured qmd judgment is narrower: + +- Retrieval quality: `tie`. ELF and qmd both pass the encoded live real-world + retrieval suite and both pass the 480-document stress retrieval baseline. +- Debug/replay ergonomics: `elf_loss`. qmd's current artifacts expose directly + inspectable top-10 JSON rows with files, line numbers, snippets, scores, and short + replay commands. ELF has stronger service traces and production-operation evidence, + but the checked-in stress report does not hydrate an equivalent candidate list. +- Expansion/fusion/rerank controls: `not_tested`. The current qmd materializer and + stress run use `--no-rerank`; no scored expansion/fusion/rerank profile exists. + +The measured OpenViking judgment is: + +- Context trajectory: `not_tested`. The pinned Docker local embedding path reaches + `add_resource`/`find`, but the same-corpus smoke remains `wrong_result` because + expected evidence terms are missed. +- Staged retrieval, hierarchy selection, and recursive/context expansion remain + `research_gate` / `not_encoded`; no ELF win, tie, or loss is claimed against those + strengths. + +## qmd Scenario Outcomes + +| Scenario | Evidence Class | Result Type | ELF Outcome | What It Means | +| --- | --- | --- | --- | --- | +| Retrieval quality | `live_real_world` | `pass` | `tie` | Both systems pass 5/5 live retrieval jobs with 6/6 expected evidence matched. | +| Local query transparency | `live_baseline_only` | `pass` | `elf_loss` | qmd exposes top-10 files, line numbers, snippets, scores, and distractor density directly in the stress artifact. | +| Expansion/fusion/rerank controls | `research_gate` | `not_encoded` | `not_tested` | No scored profile proves either system's expansion, fusion, or rerank superiority. | +| Stale context isolation | `live_real_world` | `pass` | `tie` | Both systems pass the encoded current-vs-obsolete and distractor-heavy retrieval jobs. | +| Update/delete/cold-start behavior | `live_baseline_only` | `pass` | `tie` | Equivalent update replacement, delete suppression, and cold-start recovery checks pass for both. | +| Operator-debug evidence | `live_real_world` | `not_encoded` | `not_tested` | The live sweep marks operator-debugging UX `not_encoded` for both systems. | +| Local replayability | `live_baseline_only` | `pass` | `elf_loss` | qmd has a shorter checked-in CLI replay path for the current stress profile. | +| Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` | The report classifies qmd memory-evolution failures, but qmd candidate-drop traces are not yet materialized and no pass evidence is claimed. | + +Summary: qmd strength-profile outcomes are `0` ELF wins, `3` ties, `2` ELF losses, +and `3` not-tested scenarios. This distinguishes retrieval quality from +debug/replay ergonomics: the retrieval result is tied, but the checked-in debug +artifact ergonomics currently favor qmd. + +## qmd Wrong-Result Diagnosis + +The report adds a qmd diagnosis taxonomy with four classes: + +| Diagnosis Class | Current qmd Coverage | +| --- | --- | +| `evidence_absent` | Observed on the verdict caveat, preference rationale, and delete tombstone misses. | +| `retrieved_but_dropped` | Defined but not observed because current qmd live job artifacts do not expose candidate-stage traces. | +| `selected_but_not_narrated` | Observed on supersession jobs where qmd had evidence but did not narrate current-vs-historical state. | +| `contradicted_by_lifecycle_evidence` | Observed when current, historical, supersession, or tombstone evidence keeps the answer in typed `wrong_result` state. | + +The key qmd memory-evolution diagnosis is unchanged from the June 11 diagnostic: +qmd is `0/6` pass on live memory-evolution, misses three required evidence links, +and fails the delete/TTL tombstone job. The new report records that as typed +diagnosis evidence, not as a broad ELF-over-qmd claim. + +## OpenViking Scenario Outcomes + +| Scenario | Evidence Class | Result Type | ELF Outcome | Typed Blocker | +| --- | --- | --- | --- | --- | +| Docker local embedding setup | `live_baseline_only` | `pass` | `not_tested` | none | +| Same-corpus evidence-bearing retrieval precondition | `live_baseline_only` | `wrong_result` | `not_tested` | `output_missed_expected_terms` | +| Staged retrieval trajectory | `research_gate` | `not_encoded` | `not_tested` | `needs_evidence_bearing_same_corpus_output` | +| Hierarchy selection | `research_gate` | `not_encoded` | `not_tested` | `hierarchy_output_not_scored` | +| Recursive/context expansion | `research_gate` | `not_encoded` | `not_tested` | `recursive_expansion_not_materialized` | +| Missed expected terms evidence | `live_baseline_only` | `wrong_result` | `not_tested` | `retrieval_wrong_result` | + +Summary: OpenViking context-trajectory outcomes are `0` ELF wins, `0` ties, `0` ELF +losses, and `6` not-tested scenarios. The current smoke wrong-result is useful typed +failure evidence, but it is not a scored staged-trajectory comparison. + +## Claim Boundaries + +Allowed: + +- ELF ties qmd on the current encoded retrieval-correctness surfaces. +- qmd remains stronger than ELF on the currently evidenced local query transparency + and replay artifact ergonomics. +- qmd expansion/fusion/rerank superiority is untested. +- OpenViking's Docker local embedding setup reaches runtime, but context trajectory + remains untested because evidence-bearing same-corpus retrieval is not passing. + +Not allowed: + +- Do not claim ELF broadly beats qmd. +- Do not claim qmd's debug ergonomics are equivalent to retrieval quality. +- Do not claim ELF beats OpenViking on staged retrieval, hierarchy, or recursive + context expansion. +- Do not turn `research_gate`, `not_encoded`, or `unsupported` surfaces into wins or + losses. + +## Validation Hook + +The checked-in consistency test reads the machine-readable companion report and +asserts the qmd/OpenViking scenario counts, diagnosis taxonomy, and bottom-line +claim boundaries. This keeps future report edits from silently converting untested +strength surfaces into pass claims. diff --git a/docs/guide/benchmarking/index.md b/docs/guide/benchmarking/index.md index e7b0cded..26142416 100644 --- a/docs/guide/benchmarking/index.md +++ b/docs/guide/benchmarking/index.md @@ -70,6 +70,10 @@ cleanup, use `docs/guide/single_user_production.md`. records Graphiti/Zep and Letta claim boundaries, and turns qmd, mem0/OpenMemory, Graphiti/Zep, Letta, and adjacent project strengths into benchmark-gated ELF optimization directions. +- `2026-06-11-qmd-openviking-strength-profile-report.md`: XY-899 strength-profile + report that separates qmd retrieval quality from debug/replay ergonomics, records + qmd wrong-result diagnosis classes, and preserves OpenViking context-trajectory + surfaces as `not_tested` until staged/hierarchical evidence is encoded. - `real_world_agent_memory_benchmark.md`: operator overview for the v1 real-world agent memory benchmark contract, including suite taxonomy, typed report states, knowledge-compilation fixture tasks, and the production-ops fixture target. diff --git a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json new file mode 100644 index 00000000..ee688cdd --- /dev/null +++ b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json @@ -0,0 +1,362 @@ +{ + "schema": "elf.competitor_strength_profile_report/v1", + "run_id": "2026-06-11-qmd-openviking-strength-profile", + "created_at": "2026-06-11", + "scope": "Scenario-level qmd retrieval-debug and OpenViking context-trajectory strength profile outcomes for XY-899.", + "inputs": [ + "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md", + "docs/guide/benchmarking/2026-06-11-elf-qmd-memory-evolution-diagnostic.md", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md", + "docs/spec/real_world_agent_memory_benchmark_v1.md", + "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json", + "scripts/real-world-live-adapters.sh" + ], + "outcome_terms": [ + "elf_win", + "tie", + "elf_loss", + "not_tested" + ], + "result_type_terms": [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "not_encoded", + "unsupported" + ], + "evidence_class_terms": [ + "fixture_backed", + "live_baseline_only", + "live_real_world", + "research_gate" + ], + "summary": { + "qmd": { + "overall_against_strengths": "elf_loss_on_debug_replay_ergonomics", + "retrieval_quality": "tie", + "debug_replay_ergonomics": "elf_loss", + "expansion_fusion_rerank": "not_tested", + "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior, loses the currently evidenced local debug/replay ergonomics surface, and remains untested on scored expansion, fusion, and rerank controls." + }, + "openviking": { + "overall_against_strengths": "not_tested", + "claim": "ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths. The current OpenViking Docker smoke reaches add_resource/find but is wrong_result on evidence terms, while staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." + } + }, + "qmd_strength_profile": { + "scenario_outcomes": [ + { + "scenario_id": "qmd-retrieval-quality", + "surface": "retrieval quality", + "evidence_class": "live_real_world", + "result_type": "pass", + "elf_status": "pass", + "qmd_status": "pass", + "elf_outcome": "tie", + "retrieval_quality": "ELF and qmd each pass 5/5 live real-world retrieval jobs with 6/6 expected evidence matched.", + "debug_replay_ergonomics": "not scored by this scenario", + "source_artifacts": [ + "tmp/real-world-memory/live-adapters/elf-report.json", + "tmp/real-world-memory/live-adapters/qmd-report.json", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-local-query-transparency", + "surface": "local query transparency", + "evidence_class": "live_baseline_only", + "result_type": "pass", + "elf_status": "partial", + "qmd_status": "pass", + "elf_outcome": "elf_loss", + "retrieval_quality": "not a correctness scenario", + "debug_replay_ergonomics": "qmd stress artifacts expose per-query top-10 files, line numbers, snippets, scores, and distractor density; ELF stress artifacts expose trace ids and top evidence but do not hydrate the candidate list in the checked-in report.", + "source_artifacts": [ + "scripts/live-baseline-benchmark.sh", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-expansion-fusion-rerank-controls", + "surface": "expansion, fusion, and rerank controls", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "not_encoded", + "elf_outcome": "not_tested", + "retrieval_quality": "not scored", + "debug_replay_ergonomics": "The qmd materializer and stress baseline use structured lex/vec query input with --no-rerank; no scenario scores expansion, fusion, or rerank superiority for either system.", + "source_artifacts": [ + "scripts/real-world-live-adapters.sh", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-stale-context-isolation", + "surface": "stale context isolation", + "evidence_class": "live_real_world", + "result_type": "pass", + "elf_status": "pass", + "qmd_status": "pass", + "elf_outcome": "tie", + "retrieval_quality": "Both adapters pass the encoded retrieval current-vs-obsolete and distractor-heavy jobs.", + "debug_replay_ergonomics": "The debug explanation of stale-candidate rejection is not scored beyond the job answer and evidence match.", + "source_artifacts": [ + "apps/elf-eval/fixtures/real_world_memory/retrieval/current_vs_obsolete.json", + "apps/elf-eval/fixtures/real_world_memory/retrieval/distractor_heavy.json", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-update-delete-cold-start", + "surface": "update, delete, and cold-start behavior", + "evidence_class": "live_baseline_only", + "result_type": "pass", + "elf_status": "pass", + "qmd_status": "pass", + "elf_outcome": "tie", + "retrieval_quality": "Equivalent qmd and ELF stress-baseline lifecycle checks pass for update replacement, delete suppression, and cold-start recovery.", + "debug_replay_ergonomics": "ELF has additional service lifecycle, backfill, rebuild, and resource evidence, but the equivalent qmd strength surface is a tie.", + "source_artifacts": [ + "tmp/live-baseline/live-baseline-report.json", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-operator-debug-evidence", + "surface": "operator-debug evidence", + "evidence_class": "live_real_world", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "not_encoded", + "elf_outcome": "not_tested", + "retrieval_quality": "not scored", + "debug_replay_ergonomics": "The live real-world sweep marks operator_debugging_ux not_encoded for both ELF and qmd. ELF fixture-backed operator-debug jobs pass, but they are not live adapter evidence.", + "source_artifacts": [ + "tmp/real-world-memory/live-adapters/elf-report.json", + "tmp/real-world-memory/live-adapters/qmd-report.json", + "apps/elf-eval/fixtures/real_world_memory/retrieval/stage_explainability_wrong_result.json" + ] + }, + { + "scenario_id": "qmd-local-replayability", + "surface": "local replayability", + "evidence_class": "live_baseline_only", + "result_type": "pass", + "elf_status": "partial", + "qmd_status": "pass", + "elf_outcome": "elf_loss", + "retrieval_quality": "not a correctness scenario", + "debug_replay_ergonomics": "qmd's measured replay path is collection add, update, embed -f, and query --json in a fresh CLI process; ELF has service traces and admin bundle endpoints, but the checked-in stress report does not give an equally short per-query replay artifact.", + "source_artifacts": [ + "scripts/live-baseline-benchmark.sh", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-wrong-result-diagnosis", + "surface": "wrong-result diagnosis", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "elf_status": "partial", + "qmd_status": "partial", + "elf_outcome": "not_tested", + "retrieval_quality": "The memory-evolution diagnostic classifies qmd misses and selected-but-not-narrated lifecycle failures from produced evidence; candidate-drop classification remains untested because qmd live job artifacts do not expose candidate-stage traces.", + "debug_replay_ergonomics": "The report taxonomy supports absent evidence, retrieved-but-dropped evidence, selected-but-not-narrated evidence, and lifecycle-contradicted evidence. Current qmd data exercises absent and selected-but-not-narrated classes; retrieved-but-dropped remains not observed.", + "source_artifacts": [ + "docs/guide/benchmarking/2026-06-11-elf-qmd-memory-evolution-diagnostic.md" + ] + } + ], + "win_tie_loss_summary": { + "elf_win": 0, + "tie": 3, + "elf_loss": 2, + "not_tested": 3 + }, + "wrong_result_diagnosis": { + "taxonomy": [ + { + "class": "evidence_absent", + "meaning": "Required evidence is absent from the adapter-produced evidence ids.", + "coverage": "observed" + }, + { + "class": "retrieved_but_dropped", + "meaning": "Required evidence appears in an intermediate candidate set but is absent from the final selected/narrated answer.", + "coverage": "not_observed_candidate_trace_missing" + }, + { + "class": "selected_but_not_narrated", + "meaning": "Evidence is selected or available, but the answer does not narrate the required current-vs-historical or lifecycle relationship.", + "coverage": "observed" + }, + { + "class": "contradicted_by_lifecycle_evidence", + "meaning": "The answer is contradicted or made incomplete by available current, historical, supersession, or tombstone evidence.", + "coverage": "observed" + } + ], + "jobs": [ + { + "job_id": "memory-evolution-benchmark-verdict-001", + "qmd_status": "wrong_result", + "score": 0.15, + "classifications": [ + "evidence_absent", + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [ + "verdict-bounded-private-caveat" + ], + "diagnosis": "qmd missed the caveat evidence and did not represent the superseded not-ready verdict as historical." + }, + { + "job_id": "memory-evolution-deploy-method-001", + "qmd_status": "wrong_result", + "score": 0.4, + "classifications": [ + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [], + "diagnosis": "qmd retrieved current runbook and rationale evidence, but did not preserve the old quickstart path as historical." + }, + { + "job_id": "memory-evolution-issue-state-001", + "qmd_status": "wrong_result", + "score": 0.4, + "classifications": [ + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [], + "diagnosis": "qmd found current done state and rationale evidence, but did not surface the earlier blocked state as superseded history." + }, + { + "job_id": "memory-evolution-preference-001", + "qmd_status": "wrong_result", + "score": 0.15, + "classifications": [ + "evidence_absent", + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [ + "pref-current-concise-rationale" + ], + "diagnosis": "qmd only returned rationale evidence and did not preserve the old terse preference as historical." + }, + { + "job_id": "memory-evolution-relation-temporal-001", + "qmd_status": "wrong_result", + "score": 0.35, + "classifications": [ + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [], + "diagnosis": "qmd retrieved current and historical owners, but did not produce temporal-validity explanation or update rationale." + }, + { + "job_id": "memory-evolution-delete-ttl-001", + "qmd_status": "wrong_result", + "score": 0.5, + "classifications": [ + "evidence_absent", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [ + "delete-tombstone" + ], + "diagnosis": "qmd retrieved the current plan but missed the tombstone evidence, so the delete/TTL lifecycle answer remains a typed wrong_result." + } + ] + } + }, + "openviking_context_trajectory_profile": { + "scenario_outcomes": [ + { + "scenario_id": "openviking-local-embed-setup", + "surface": "Docker local embedding setup", + "evidence_class": "live_baseline_only", + "result_type": "pass", + "openviking_status": "pass", + "elf_equivalent_status": "unsupported", + "elf_outcome": "not_tested", + "typed_blocker": null, + "evidence": "The pinned llama-cpp-python==0.3.28 CPU wheel path installed and OpenViking reached add_resource/find in Docker." + }, + { + "scenario_id": "openviking-evidence-bearing-retrieval-precondition", + "surface": "same-corpus evidence-bearing retrieval precondition", + "evidence_class": "live_baseline_only", + "result_type": "wrong_result", + "openviking_status": "wrong_result", + "elf_equivalent_status": "pass", + "elf_outcome": "not_tested", + "typed_blocker": "output_missed_expected_terms", + "evidence": "OpenViking add_resource/find returned resources but matched 0/3 expected evidence-term checks; this is a wrong_result smoke output, not a trajectory comparison." + }, + { + "scenario_id": "openviking-staged-retrieval-trajectory", + "surface": "staged retrieval trajectory", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "openviking_status": "not_encoded", + "elf_equivalent_status": "not_encoded", + "elf_outcome": "not_tested", + "typed_blocker": "needs_evidence_bearing_same_corpus_output", + "evidence": "No stage trajectory scoring is claimed until OpenViking returns evidence-bearing same-corpus output." + }, + { + "scenario_id": "openviking-hierarchy-selection", + "surface": "hierarchy selection", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "openviking_status": "not_encoded", + "elf_equivalent_status": "unsupported", + "elf_outcome": "not_tested", + "typed_blocker": "hierarchy_output_not_scored", + "evidence": "The viking:// hierarchy model remains a reference strength, but no real_world_job output scores hierarchy selection." + }, + { + "scenario_id": "openviking-recursive-context-expansion", + "surface": "recursive/context expansion", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "openviking_status": "not_encoded", + "elf_equivalent_status": "not_encoded", + "elf_outcome": "not_tested", + "typed_blocker": "recursive_expansion_not_materialized", + "evidence": "Recursive/context expansion remains unmaterialized in the Docker adapter; no pass/fail quality claim is allowed." + }, + { + "scenario_id": "openviking-missed-expected-terms-evidence", + "surface": "typed failure evidence when expected terms are missed", + "evidence_class": "live_baseline_only", + "result_type": "wrong_result", + "openviking_status": "wrong_result", + "elf_equivalent_status": "pass", + "elf_outcome": "not_tested", + "typed_blocker": "retrieval_wrong_result", + "evidence": "The baseline report preserves missed expected terms as wrong_result instead of loosening evidence expectations or reporting setup failure." + } + ], + "win_tie_loss_summary": { + "elf_win": 0, + "tie": 0, + "elf_loss": 0, + "not_tested": 6 + } + }, + "claim_boundaries": [ + "ELF does not broadly beat qmd; it ties retrieval correctness but loses the measured debug/replay ergonomics surface.", + "qmd expansion, fusion, and rerank superiority remains not_tested because the current qmd paths use --no-rerank and do not score internals.", + "ELF does not beat OpenViking on context trajectory; OpenViking trajectory strengths remain not_tested behind a wrong_result same-corpus output precondition.", + "Research_gate records are follow-up gates, not pass evidence.", + "Missing equivalent surfaces are encoded as unsupported or not_encoded rather than fake losses." + ] +} diff --git a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json index b847ecc7..c525de3f 100644 --- a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json +++ b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json @@ -117,7 +117,7 @@ "unsupported_or_blocked_status": { "state": "not_encoded", "typed_reason": "deep_profile_and_non_retrieval_suites_not_encoded", - "details": "The full live sweep passes targeted retrieval suites but keeps memory_evolution wrong_result and several broader suites not_encoded or incomplete." + "details": "The full live sweep passes targeted retrieval suites but keeps memory_evolution wrong_result and several broader suites not_encoded or blocked." }, "benchmark_before_claim": "Run qmd deep retrieval/debug profile and full-suite live real-world replay with trace-level diagnostics before claiming ELF wins, ties, or loses on retrieval debugging.", "borrow_if_stronger": "Borrow transparent local knobs for query rewriting, weighted fusion, rerank explanation, and command-line replay." @@ -509,9 +509,9 @@ { "scenario_id": "production_ops", "scenario": "production ops", - "current_elf_evidence": "ELF production runbooks and fixture production_ops cover restore, Qdrant rebuild, backfill resume, resource envelope, and typed private/credential blockers; live_real_world production_ops is incomplete.", + "current_elf_evidence": "ELF production runbooks and fixture production_ops cover restore, Qdrant rebuild, backfill resume, resource envelope, and typed private/credential blockers; live_real_world production_ops is blocked.", "strongest_competitor_or_reference": "ELF production gate, qmd, RAG/RAGFlow resource gates", - "current_competitor_evidence": "qmd live production_ops is incomplete; RAGFlow/GraphRAG/LightRAG resource gates are research_gate blocked.", + "current_competitor_evidence": "qmd live production_ops is blocked; RAGFlow/GraphRAG/LightRAG resource gates are research_gate blocked.", "current_state": "ELF has the strongest checked-in production evidence, but private corpus and credentialed gates remain blocked.", "next_measurement": "Rerun private-corpus and credentialed production-ops gates only when operator-owned manifest and credentials are supplied." }, From 533b51ffdb6ad4cc4454840db5a20c01972ffee4 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 12:05:45 +0800 Subject: [PATCH 02/10] {"schema":"decodex/commit/1","summary":"Repair qmd and OpenViking strength-profile report consistency","authority":"XY-899"} --- .../tests/real_world_job_benchmark.rs | 68 ++++++++++++++++++- ...-qmd-openviking-strength-profile-report.md | 27 +++++--- ...06-11-elf-qmd-retrieval-debug-profile.json | 8 +-- ...2026-06-11-measurement-coverage-audit.json | 16 ++--- ...md-openviking-strength-profile-report.json | 12 ++-- 5 files changed, 101 insertions(+), 30 deletions(-) diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 154f8ec3..4744aa7c 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -93,6 +93,20 @@ fn measurement_coverage_audit_path() -> Result { .join("2026-06-11-measurement-coverage-audit.md")) } +fn measurement_coverage_audit_json_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-measurement-coverage-audit.json")) +} + +fn retrieval_debug_profile_json_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-elf-qmd-retrieval-debug-profile.json")) +} + fn competitor_strength_matrix_path() -> Result { Ok(workspace_root()? .join("docs") @@ -839,8 +853,13 @@ fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result #[test] fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { let measurement_audit = fs::read_to_string(measurement_coverage_audit_path()?)?; + let measurement_audit_json = serde_json::from_str::(&fs::read_to_string( + measurement_coverage_audit_json_path()?, + )?)?; let competitor_matrix = fs::read_to_string(competitor_strength_matrix_path()?)?; let external_manifest = fs::read_to_string(external_adapter_manifest_path())?; + let retrieval_debug_profile = + serde_json::from_str::(&fs::read_to_string(retrieval_debug_profile_json_path()?)?)?; assert!( measurement_audit.contains( @@ -871,6 +890,44 @@ fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { assert!(!external_manifest.contains(stale_phrase)); } + let qmd_live = find_by_field( + array_at(&measurement_audit_json, "/live_real_world_adapters")?, + "/adapter", + "qmd live CLI adapter", + )?; + + assert_eq!(qmd_live.pointer("/pass").and_then(Value::as_u64), Some(17)); + assert_eq!(qmd_live.pointer("/wrong_result").and_then(Value::as_u64), Some(6)); + assert_eq!(qmd_live.pointer("/expected_evidence_matched").and_then(Value::as_u64), Some(38)); + assert_eq!(qmd_live.pointer("/evidence_covered_count").and_then(Value::as_u64), Some(45)); + + let memory_evolution = find_by_field( + array_at(&measurement_audit_json, "/live_suite_breakdown")?, + "/suite", + "memory_evolution", + )?; + + assert_eq!( + memory_evolution.pointer("/elf_status_counts/wrong_result").and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + memory_evolution.pointer("/qmd_status_counts/wrong_result").and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + retrieval_debug_profile + .pointer("/live_real_world_full_sweep_context/qmd/pass") + .and_then(Value::as_u64), + Some(17) + ); + assert_eq!( + retrieval_debug_profile + .pointer("/live_real_world_full_sweep_context/qmd/wrong_result") + .and_then(Value::as_u64), + Some(6) + ); + Ok(()) } @@ -889,7 +946,7 @@ fn assert_strength_profile_summary(report: &Value) { ); assert_eq!( report.pointer("/summary/openviking/overall_against_strengths").and_then(Value::as_str), - Some("not_tested") + Some("not_tested_on_context_trajectory") ); assert_eq!( report @@ -917,7 +974,13 @@ fn assert_strength_profile_summary(report: &Value) { report .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/not_tested") .and_then(Value::as_u64), - Some(6) + Some(4) + ); + assert_eq!( + report + .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/elf_win") + .and_then(Value::as_u64), + Some(2) ); } @@ -1002,6 +1065,7 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { ); assert_eq!(trajectory.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); assert_eq!(precondition.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); + assert_eq!(precondition.pointer("/elf_outcome").and_then(Value::as_str), Some("elf_win")); assert_eq!( precondition.pointer("/typed_blocker").and_then(Value::as_str), Some("output_missed_expected_terms") diff --git a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md index b0375c15..b788d7d4 100644 --- a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md +++ b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md @@ -28,11 +28,14 @@ The measured qmd judgment is narrower: - Expansion/fusion/rerank controls: `not_tested`. The current qmd materializer and stress run use `--no-rerank`; no scored expansion/fusion/rerank profile exists. -The measured OpenViking judgment is: - -- Context trajectory: `not_tested`. The pinned Docker local embedding path reaches - `add_resource`/`find`, but the same-corpus smoke remains `wrong_result` because - expected evidence terms are missed. +The measured OpenViking judgment is split by surface: + +- Same-corpus evidence-bearing preconditions: `elf_win`. The pinned Docker local + embedding path reaches `add_resource`/`find`, but the OpenViking smoke remains + `wrong_result` because expected evidence terms are missed while ELF passes the + equivalent retrieval precondition. +- Context trajectory strengths: `not_tested`. The current OpenViking wrong-result + smoke is not a scored staged-trajectory comparison. - Staged retrieval, hierarchy selection, and recursive/context expansion remain `research_gate` / `not_encoded`; no ELF win, tie, or loss is claimed against those strengths. @@ -76,15 +79,17 @@ diagnosis evidence, not as a broad ELF-over-qmd claim. | Scenario | Evidence Class | Result Type | ELF Outcome | Typed Blocker | | --- | --- | --- | --- | --- | | Docker local embedding setup | `live_baseline_only` | `pass` | `not_tested` | none | -| Same-corpus evidence-bearing retrieval precondition | `live_baseline_only` | `wrong_result` | `not_tested` | `output_missed_expected_terms` | +| Same-corpus evidence-bearing retrieval precondition | `live_baseline_only` | `wrong_result` | `elf_win` | `output_missed_expected_terms` | | Staged retrieval trajectory | `research_gate` | `not_encoded` | `not_tested` | `needs_evidence_bearing_same_corpus_output` | | Hierarchy selection | `research_gate` | `not_encoded` | `not_tested` | `hierarchy_output_not_scored` | | Recursive/context expansion | `research_gate` | `not_encoded` | `not_tested` | `recursive_expansion_not_materialized` | -| Missed expected terms evidence | `live_baseline_only` | `wrong_result` | `not_tested` | `retrieval_wrong_result` | +| Missed expected terms evidence | `live_baseline_only` | `wrong_result` | `elf_win` | `retrieval_wrong_result` | -Summary: OpenViking context-trajectory outcomes are `0` ELF wins, `0` ties, `0` ELF -losses, and `6` not-tested scenarios. The current smoke wrong-result is useful typed -failure evidence, but it is not a scored staged-trajectory comparison. +Summary: OpenViking profile outcomes are `2` ELF wins, `0` ties, `0` ELF losses, and +`4` not-tested scenarios. The two wins are only same-corpus evidence-bearing +preconditions and missed-term failure evidence. The current smoke wrong-result is +useful typed failure evidence, but it is not a scored staged-trajectory comparison, +so context-trajectory strengths remain not tested. ## Claim Boundaries @@ -96,6 +101,8 @@ Allowed: - qmd expansion/fusion/rerank superiority is untested. - OpenViking's Docker local embedding setup reaches runtime, but context trajectory remains untested because evidence-bearing same-corpus retrieval is not passing. +- ELF currently wins only the equivalent OpenViking same-corpus retrieval + precondition surfaces, not OpenViking's staged trajectory strengths. Not allowed: diff --git a/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json b/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json index fed5fed9..72f22936 100644 --- a/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json +++ b/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json @@ -48,12 +48,12 @@ }, "qmd": { "job_count": 38, - "pass": 18, - "wrong_result": 5, + "pass": 17, + "wrong_result": 6, "blocked": 2, "not_encoded": 13, - "mean_score": 0.512, - "mean_latency_ms": 705.877 + "mean_score": 0.486, + "mean_latency_ms": 691.163 } }, "stress_baseline": { diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index b04d86ef..575bdf6b 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -55,16 +55,16 @@ "adapter": "qmd live CLI adapter", "job_count": 38, "encoded_suite_count": 11, - "pass": 18, - "wrong_result": 5, + "pass": 17, + "wrong_result": 6, "blocked": 2, "not_encoded": 13, - "mean_score": 0.512, - "mean_latency_ms": 719.758, + "mean_score": 0.486, + "mean_latency_ms": 691.163, "expected_evidence_total": 77, - "expected_evidence_matched": 41, + "expected_evidence_matched": 38, "evidence_required_count": 84, - "evidence_covered_count": 48 + "evidence_covered_count": 45 } ], "live_suite_breakdown": [ @@ -73,7 +73,7 @@ {"suite": "retrieval", "jobs": 5, "status_counts": {"pass": 5}}, {"suite": "project_decisions", "jobs": 5, "status_counts": {"pass": 5}}, {"suite": "personalization", "jobs": 1, "status_counts": {"pass": 1}}, - {"suite": "memory_evolution", "jobs": 6, "status_counts": {"pass": 1, "wrong_result": 5}}, + {"suite": "memory_evolution", "jobs": 6, "elf_status_counts": {"pass": 1, "wrong_result": 5}, "qmd_status_counts": {"wrong_result": 6}}, {"suite": "capture_integration", "jobs": 2, "status_counts": {"not_encoded": 2}}, {"suite": "consolidation", "jobs": 4, "status_counts": {"not_encoded": 4}}, {"suite": "knowledge_compilation", "jobs": 2, "status_counts": {"not_encoded": 2}}, @@ -99,7 +99,7 @@ } }, "claim_boundary": { - "elf_vs_qmd": "tie_on_current_encoded_live_real_world_shape_not_overall_win", + "elf_vs_qmd": "narrow_elf_lead_from_delete_ttl_not_overall_win", "elf_personal_production": "credible_with_bounded_caveats", "broad_competitor_superiority": "not_proven", "major_unmeasured_strengths": [ diff --git a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json index ee688cdd..3112b271 100644 --- a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json +++ b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json @@ -40,8 +40,8 @@ "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior, loses the currently evidenced local debug/replay ergonomics surface, and remains untested on scored expansion, fusion, and rerank controls." }, "openviking": { - "overall_against_strengths": "not_tested", - "claim": "ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths. The current OpenViking Docker smoke reaches add_resource/find but is wrong_result on evidence terms, while staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." + "overall_against_strengths": "not_tested_on_context_trajectory", + "claim": "ELF has measured wins only on same-corpus evidence-bearing preconditions where OpenViking currently returns wrong_result. ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths because staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." } }, "qmd_strength_profile": { @@ -296,7 +296,7 @@ "result_type": "wrong_result", "openviking_status": "wrong_result", "elf_equivalent_status": "pass", - "elf_outcome": "not_tested", + "elf_outcome": "elf_win", "typed_blocker": "output_missed_expected_terms", "evidence": "OpenViking add_resource/find returned resources but matched 0/3 expected evidence-term checks; this is a wrong_result smoke output, not a trajectory comparison." }, @@ -340,16 +340,16 @@ "result_type": "wrong_result", "openviking_status": "wrong_result", "elf_equivalent_status": "pass", - "elf_outcome": "not_tested", + "elf_outcome": "elf_win", "typed_blocker": "retrieval_wrong_result", "evidence": "The baseline report preserves missed expected terms as wrong_result instead of loosening evidence expectations or reporting setup failure." } ], "win_tie_loss_summary": { - "elf_win": 0, + "elf_win": 2, "tie": 0, "elf_loss": 0, - "not_tested": 6 + "not_tested": 4 } }, "claim_boundaries": [ From 50bbe39a2c3faf0a7cd17c4b067a0af40e75eabf Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 12:12:52 +0800 Subject: [PATCH 03/10] {"schema":"decodex/commit/1","summary":"Repair benchmark summary report date heading","authority":"XY-899"} --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 95285c66..723dd465 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ Detailed evidence and interpretation: live sweep, but that sweep still contains typed non-pass states and is not full-suite parity. -Evidence-backed position after the June 10 real-world report: +Evidence-backed position after the June 11 real-world reports: - ELF is better evidenced than the tested alternatives on evidence-bound writes, deterministic ingestion boundaries, Postgres source-of-truth plus rebuildable Qdrant From 28a7151af906bf4d9fdc5a09a875b33c6596d69f Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 12:24:12 +0800 Subject: [PATCH 04/10] {"schema":"decodex/commit/1","summary":"Tighten strength-profile claim-boundary assertions","authority":"XY-899"} --- .../tests/real_world_job_benchmark.rs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 4744aa7c..6a2ab241 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -845,6 +845,7 @@ fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result assert_qmd_strength_profile(&report)?; assert_qmd_wrong_result_diagnosis(&report)?; assert_openviking_strength_profile(&report)?; + assert_strength_profile_json_claim_boundaries(&report)?; assert_strength_profile_markdown_boundaries(&markdown); Ok(()) @@ -1074,12 +1075,56 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { Ok(()) } +fn assert_strength_profile_json_claim_boundaries(report: &Value) -> Result<()> { + assert!(array_contains_str( + report, + "/claim_boundaries", + "ELF does not broadly beat qmd; it ties retrieval correctness but loses the measured debug/replay ergonomics surface." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "qmd expansion, fusion, and rerank superiority remains not_tested because the current qmd paths use --no-rerank and do not score internals." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "ELF does not beat OpenViking on context trajectory; OpenViking trajectory strengths remain not_tested behind a wrong_result same-corpus output precondition." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "Research_gate records are follow-up gates, not pass evidence." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "Missing equivalent surfaces are encoded as unsupported or not_encoded rather than fake losses." + )?); + + Ok(()) +} + fn assert_strength_profile_markdown_boundaries(markdown: &str) { assert!( markdown.contains( "| Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` |" ) ); + assert!( + markdown.contains("ELF ties qmd on the current encoded retrieval-correctness surfaces") + ); + assert!(markdown.contains( + "qmd remains stronger than ELF on the currently evidenced local query transparency" + )); + assert!(markdown.contains("ELF currently wins only the equivalent OpenViking same-corpus")); + assert!(markdown.contains("Do not claim ELF broadly beats qmd")); + assert!(markdown.contains( + "Do not claim ELF beats OpenViking on staged retrieval, hierarchy, or recursive" + )); + assert!(markdown.contains( + "Do not turn `research_gate`, `not_encoded`, or `unsupported` surfaces into wins" + )); assert!(markdown.contains("no pass evidence is claimed")); assert!(markdown.contains("typed `wrong_result` state")); } From f71253408f68090e7dd59c0a03a5cba1fda5f0dc Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:00:24 +0800 Subject: [PATCH 05/10] {"schema":"decodex/commit/1","summary":"Repair strength-profile claim boundaries","authority":"XY-899"} --- .../memory_projects_manifest.json | 2 +- .../tests/real_world_job_benchmark.rs | 68 +++++++++++++++++-- ...-qmd-openviking-strength-profile-report.md | 37 +++++----- ...md-openviking-strength-profile-report.json | 50 ++++++++------ 4 files changed, 114 insertions(+), 43 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 33e15d4b..4b0cb84e 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -933,7 +933,7 @@ { "capability": "real_world_job_adapter", "status": "not_encoded", - "evidence": "The qmd live real-world slice covers representative jobs only; expanded retrieval-debug suites need their own materialized adapter run." + "evidence": "The qmd live real-world sweep covers the current encoded fixture corpus; expanded retrieval-debug strength suites still need their own materialized adapter run." }, { "capability": "host_global_install_boundary", diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 6a2ab241..2610da22 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -842,6 +842,7 @@ fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result let markdown = fs::read_to_string(strength_profile_markdown_path()?)?; assert_strength_profile_summary(&report); + assert_strength_profile_terms(&report)?; assert_qmd_strength_profile(&report)?; assert_qmd_wrong_result_diagnosis(&report)?; assert_openviking_strength_profile(&report)?; @@ -878,6 +879,9 @@ fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { assert!(external_manifest.contains( "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible." )); + assert!(external_manifest.contains( + "The qmd live real-world sweep covers the current encoded fixture corpus; expanded retrieval-debug strength suites still need their own materialized adapter run." + )); for stale_phrase in [ "same live sweep shape as ELF", @@ -885,6 +889,7 @@ fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { "both systems currently fail 5/6 live memory-evolution jobs", "wrong_result, incomplete, blocked, and not_encoded states remain visible", "broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`", + "The qmd live real-world slice covers representative jobs only", ] { assert!(!measurement_audit.contains(stale_phrase)); assert!(!competitor_matrix.contains(stale_phrase)); @@ -942,9 +947,13 @@ fn assert_strength_profile_summary(report: &Value) { Some("tie") ); assert_eq!( - report.pointer("/summary/qmd/debug_replay_ergonomics").and_then(Value::as_str), + report.pointer("/summary/qmd/local_query_transparency").and_then(Value::as_str), Some("elf_loss") ); + assert_eq!( + report.pointer("/summary/qmd/local_replayability").and_then(Value::as_str), + Some("not_tested") + ); assert_eq!( report.pointer("/summary/openviking/overall_against_strengths").and_then(Value::as_str), Some("not_tested_on_context_trajectory") @@ -963,28 +972,64 @@ fn assert_strength_profile_summary(report: &Value) { report .pointer("/qmd_strength_profile/win_tie_loss_summary/elf_loss") .and_then(Value::as_u64), - Some(2) + Some(1) ); assert_eq!( report .pointer("/qmd_strength_profile/win_tie_loss_summary/not_tested") .and_then(Value::as_u64), - Some(3) + Some(4) ); assert_eq!( report .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/not_tested") .and_then(Value::as_u64), - Some(4) + Some(5) ); assert_eq!( report .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/elf_win") .and_then(Value::as_u64), - Some(2) + Some(1) ); } +fn assert_strength_profile_terms(report: &Value) -> Result<()> { + let result_terms = array_at(report, "/result_type_terms")?; + let coverage_terms = array_at(report, "/coverage_status_terms")?; + + assert!(!result_terms.iter().any(|term| term.as_str() == Some("unsupported"))); + assert!(result_terms.iter().any(|term| term.as_str() == Some("unsupported_claim"))); + assert!(coverage_terms.iter().any(|term| term.as_str() == Some("unsupported"))); + + for scenario in array_at(report, "/qmd_strength_profile/scenario_outcomes")? { + assert_value_in_terms(scenario, "/result_type", result_terms)?; + assert_value_in_terms(scenario, "/elf_status", coverage_terms)?; + assert_value_in_terms(scenario, "/qmd_status", coverage_terms)?; + } + for scenario in array_at(report, "/openviking_context_trajectory_profile/scenario_outcomes")? { + assert_value_in_terms(scenario, "/result_type", result_terms)?; + assert_value_in_terms(scenario, "/openviking_status", coverage_terms)?; + assert_value_in_terms(scenario, "/elf_equivalent_status", coverage_terms)?; + } + + Ok(()) +} + +fn assert_value_in_terms(value: &Value, pointer: &str, terms: &[Value]) -> Result<()> { + let actual = value + .pointer(pointer) + .and_then(Value::as_str) + .ok_or_else(|| eyre::eyre!("missing string at {pointer}"))?; + + assert!( + terms.iter().any(|term| term.as_str() == Some(actual)), + "{actual} at {pointer} is not declared in the report term list" + ); + + Ok(()) +} + fn assert_qmd_strength_profile(report: &Value) -> Result<()> { let qmd_scenarios = array_at(report, "/qmd_strength_profile/scenario_outcomes")?; let local_transparency = @@ -992,6 +1037,7 @@ fn assert_qmd_strength_profile(report: &Value) -> Result<()> { let retrieval = find_by_field(qmd_scenarios, "/scenario_id", "qmd-retrieval-quality")?; let rerank_controls = find_by_field(qmd_scenarios, "/scenario_id", "qmd-expansion-fusion-rerank-controls")?; + let replayability = find_by_field(qmd_scenarios, "/scenario_id", "qmd-local-replayability")?; let wrong_result = find_by_field(qmd_scenarios, "/scenario_id", "qmd-wrong-result-diagnosis")?; assert_eq!(qmd_scenarios.len(), 8); @@ -1004,6 +1050,8 @@ fn assert_qmd_strength_profile(report: &Value) -> Result<()> { rerank_controls.pointer("/result_type").and_then(Value::as_str), Some("not_encoded") ); + assert_eq!(replayability.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(replayability.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); assert_eq!( wrong_result.pointer("/evidence_class").and_then(Value::as_str), Some("research_gate") @@ -1058,6 +1106,11 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { "/scenario_id", "openviking-evidence-bearing-retrieval-precondition", )?; + let missed_terms = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-missed-expected-terms-evidence", + )?; assert_eq!(openviking_scenarios.len(), 6); assert_eq!( @@ -1071,6 +1124,8 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { precondition.pointer("/typed_blocker").and_then(Value::as_str), Some("output_missed_expected_terms") ); + assert_eq!(missed_terms.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); + assert_eq!(missed_terms.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); Ok(()) } @@ -1079,7 +1134,7 @@ fn assert_strength_profile_json_claim_boundaries(report: &Value) -> Result<()> { assert!(array_contains_str( report, "/claim_boundaries", - "ELF does not broadly beat qmd; it ties retrieval correctness but loses the measured debug/replay ergonomics surface." + "ELF does not broadly beat qmd; it ties retrieval correctness, loses the measured query-transparency surface, and leaves replayability not_tested." )?); assert!(array_contains_str( report, @@ -1117,6 +1172,7 @@ fn assert_strength_profile_markdown_boundaries(markdown: &str) { assert!(markdown.contains( "qmd remains stronger than ELF on the currently evidenced local query transparency" )); + assert!(markdown.contains("replayability remains unscored")); assert!(markdown.contains("ELF currently wins only the equivalent OpenViking same-corpus")); assert!(markdown.contains("Do not claim ELF broadly beats qmd")); assert!(markdown.contains( diff --git a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md index b788d7d4..37c9c0e6 100644 --- a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md +++ b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md @@ -21,10 +21,13 @@ The measured qmd judgment is narrower: - Retrieval quality: `tie`. ELF and qmd both pass the encoded live real-world retrieval suite and both pass the 480-document stress retrieval baseline. -- Debug/replay ergonomics: `elf_loss`. qmd's current artifacts expose directly - inspectable top-10 JSON rows with files, line numbers, snippets, scores, and short - replay commands. ELF has stronger service traces and production-operation evidence, - but the checked-in stress report does not hydrate an equivalent candidate list. +- Local query transparency: `elf_loss`. qmd's current artifacts expose directly + inspectable top-10 JSON rows with files, line numbers, snippets, and scores. ELF + has stronger service traces and production-operation evidence, but the checked-in + stress report does not hydrate an equivalent candidate list. +- Local replayability: `not_tested`. qmd has a concise observed CLI replay path, and + ELF has service traces plus admin bundle endpoints, but no scored replayability rule + compares those surfaces yet. - Expansion/fusion/rerank controls: `not_tested`. The current qmd materializer and stress run use `--no-rerank`; no scored expansion/fusion/rerank profile exists. @@ -50,13 +53,13 @@ The measured OpenViking judgment is split by surface: | Stale context isolation | `live_real_world` | `pass` | `tie` | Both systems pass the encoded current-vs-obsolete and distractor-heavy retrieval jobs. | | Update/delete/cold-start behavior | `live_baseline_only` | `pass` | `tie` | Equivalent update replacement, delete suppression, and cold-start recovery checks pass for both. | | Operator-debug evidence | `live_real_world` | `not_encoded` | `not_tested` | The live sweep marks operator-debugging UX `not_encoded` for both systems. | -| Local replayability | `live_baseline_only` | `pass` | `elf_loss` | qmd has a shorter checked-in CLI replay path for the current stress profile. | +| Local replayability | `live_baseline_only` | `not_encoded` | `not_tested` | qmd has a shorter observed CLI replay path, but no scored replayability rule compares it with ELF's trace/admin replay surfaces yet. | | Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` | The report classifies qmd memory-evolution failures, but qmd candidate-drop traces are not yet materialized and no pass evidence is claimed. | -Summary: qmd strength-profile outcomes are `0` ELF wins, `3` ties, `2` ELF losses, -and `3` not-tested scenarios. This distinguishes retrieval quality from -debug/replay ergonomics: the retrieval result is tied, but the checked-in debug -artifact ergonomics currently favor qmd. +Summary: qmd strength-profile outcomes are `0` ELF wins, `3` ties, `1` ELF loss, +and `4` not-tested scenarios. This distinguishes retrieval quality from +debug/replay ergonomics: the retrieval result is tied, the checked-in query-debug +artifact ergonomics currently favor qmd, and replayability remains unscored. ## qmd Wrong-Result Diagnosis @@ -83,13 +86,13 @@ diagnosis evidence, not as a broad ELF-over-qmd claim. | Staged retrieval trajectory | `research_gate` | `not_encoded` | `not_tested` | `needs_evidence_bearing_same_corpus_output` | | Hierarchy selection | `research_gate` | `not_encoded` | `not_tested` | `hierarchy_output_not_scored` | | Recursive/context expansion | `research_gate` | `not_encoded` | `not_tested` | `recursive_expansion_not_materialized` | -| Missed expected terms evidence | `live_baseline_only` | `wrong_result` | `elf_win` | `retrieval_wrong_result` | +| Missed expected terms evidence | `live_baseline_only` | `wrong_result` | `not_tested` | `retrieval_wrong_result` | -Summary: OpenViking profile outcomes are `2` ELF wins, `0` ties, `0` ELF losses, and -`4` not-tested scenarios. The two wins are only same-corpus evidence-bearing -preconditions and missed-term failure evidence. The current smoke wrong-result is -useful typed failure evidence, but it is not a scored staged-trajectory comparison, -so context-trajectory strengths remain not tested. +Summary: OpenViking profile outcomes are `1` ELF win, `0` ties, `0` ELF losses, and +`5` not-tested scenarios. The single win is only the same-corpus evidence-bearing +precondition. The current smoke wrong-result is useful typed failure evidence, but it +is not a second comparative win and not a scored staged-trajectory comparison, so +context-trajectory strengths remain not tested. ## Claim Boundaries @@ -97,12 +100,12 @@ Allowed: - ELF ties qmd on the current encoded retrieval-correctness surfaces. - qmd remains stronger than ELF on the currently evidenced local query transparency - and replay artifact ergonomics. + artifact ergonomics; replayability is observed but not scored. - qmd expansion/fusion/rerank superiority is untested. - OpenViking's Docker local embedding setup reaches runtime, but context trajectory remains untested because evidence-bearing same-corpus retrieval is not passing. - ELF currently wins only the equivalent OpenViking same-corpus retrieval - precondition surfaces, not OpenViking's staged trajectory strengths. + precondition surface, not OpenViking's staged trajectory strengths. Not allowed: diff --git a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json index 3112b271..70b39131 100644 --- a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json +++ b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json @@ -22,8 +22,19 @@ "wrong_result", "blocked", "incomplete", + "lifecycle_fail", "not_encoded", - "unsupported" + "unsupported_claim" + ], + "coverage_status_terms": [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported", + "unsupported_claim" ], "evidence_class_terms": [ "fixture_backed", @@ -33,15 +44,16 @@ ], "summary": { "qmd": { - "overall_against_strengths": "elf_loss_on_debug_replay_ergonomics", + "overall_against_strengths": "elf_loss_on_query_transparency_replay_untested", "retrieval_quality": "tie", - "debug_replay_ergonomics": "elf_loss", + "local_query_transparency": "elf_loss", + "local_replayability": "not_tested", "expansion_fusion_rerank": "not_tested", - "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior, loses the currently evidenced local debug/replay ergonomics surface, and remains untested on scored expansion, fusion, and rerank controls." + "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior, loses the currently evidenced local query-transparency surface, and remains untested on scored replayability, expansion, fusion, and rerank controls." }, "openviking": { "overall_against_strengths": "not_tested_on_context_trajectory", - "claim": "ELF has measured wins only on same-corpus evidence-bearing preconditions where OpenViking currently returns wrong_result. ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths because staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." + "claim": "ELF has one measured win on the same-corpus evidence-bearing precondition where OpenViking currently returns wrong_result. ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths because staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." } }, "qmd_strength_profile": { @@ -67,7 +79,7 @@ "surface": "local query transparency", "evidence_class": "live_baseline_only", "result_type": "pass", - "elf_status": "partial", + "elf_status": "not_encoded", "qmd_status": "pass", "elf_outcome": "elf_loss", "retrieval_quality": "not a correctness scenario", @@ -143,12 +155,12 @@ "scenario_id": "qmd-local-replayability", "surface": "local replayability", "evidence_class": "live_baseline_only", - "result_type": "pass", - "elf_status": "partial", + "result_type": "not_encoded", + "elf_status": "not_encoded", "qmd_status": "pass", - "elf_outcome": "elf_loss", + "elf_outcome": "not_tested", "retrieval_quality": "not a correctness scenario", - "debug_replay_ergonomics": "qmd's measured replay path is collection add, update, embed -f, and query --json in a fresh CLI process; ELF has service traces and admin bundle endpoints, but the checked-in stress report does not give an equally short per-query replay artifact.", + "debug_replay_ergonomics": "qmd's observed replay path is collection add, update, embed -f, and query --json in a fresh CLI process; ELF has service traces and admin bundle endpoints, but no scored replayability rule compares the two surfaces yet.", "source_artifacts": [ "scripts/live-baseline-benchmark.sh", "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" @@ -159,8 +171,8 @@ "surface": "wrong-result diagnosis", "evidence_class": "research_gate", "result_type": "not_encoded", - "elf_status": "partial", - "qmd_status": "partial", + "elf_status": "not_encoded", + "qmd_status": "not_encoded", "elf_outcome": "not_tested", "retrieval_quality": "The memory-evolution diagnostic classifies qmd misses and selected-but-not-narrated lifecycle failures from produced evidence; candidate-drop classification remains untested because qmd live job artifacts do not expose candidate-stage traces.", "debug_replay_ergonomics": "The report taxonomy supports absent evidence, retrieved-but-dropped evidence, selected-but-not-narrated evidence, and lifecycle-contradicted evidence. Current qmd data exercises absent and selected-but-not-narrated classes; retrieved-but-dropped remains not observed.", @@ -172,8 +184,8 @@ "win_tie_loss_summary": { "elf_win": 0, "tie": 3, - "elf_loss": 2, - "not_tested": 3 + "elf_loss": 1, + "not_tested": 4 }, "wrong_result_diagnosis": { "taxonomy": [ @@ -340,20 +352,20 @@ "result_type": "wrong_result", "openviking_status": "wrong_result", "elf_equivalent_status": "pass", - "elf_outcome": "elf_win", + "elf_outcome": "not_tested", "typed_blocker": "retrieval_wrong_result", - "evidence": "The baseline report preserves missed expected terms as wrong_result instead of loosening evidence expectations or reporting setup failure." + "evidence": "The baseline report preserves the same missed expected terms as wrong_result instead of loosening evidence expectations or reporting setup failure; this row documents typed failure evidence and is not counted as a second comparative win." } ], "win_tie_loss_summary": { - "elf_win": 2, + "elf_win": 1, "tie": 0, "elf_loss": 0, - "not_tested": 4 + "not_tested": 5 } }, "claim_boundaries": [ - "ELF does not broadly beat qmd; it ties retrieval correctness but loses the measured debug/replay ergonomics surface.", + "ELF does not broadly beat qmd; it ties retrieval correctness, loses the measured query-transparency surface, and leaves replayability not_tested.", "qmd expansion, fusion, and rerank superiority remains not_tested because the current qmd paths use --no-rerank and do not score internals.", "ELF does not beat OpenViking on context trajectory; OpenViking trajectory strengths remain not_tested behind a wrong_result same-corpus output precondition.", "Research_gate records are follow-up gates, not pass evidence.", From dc9187cb811a2ced874a1eedea99f041541d7613 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:13:35 +0800 Subject: [PATCH 06/10] {"schema":"decodex/commit/1","summary":"Normalize strength-profile summary outcomes","authority":"XY-899"} --- .../tests/real_world_job_benchmark.rs | 77 ++++++++++++++++++- ...on-direction-from-competitor-benchmarks.md | 5 +- ...md-openviking-strength-profile-report.json | 6 +- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 2610da22..8c8b18b7 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -160,6 +160,17 @@ fn array_contains_str(value: &Value, pointer: &str, expected: &str) -> Result Result> { + array_at(value, pointer)? + .iter() + .map(|item| { + item.as_str() + .map(str::to_owned) + .ok_or_else(|| eyre::eyre!("non-string entry at {pointer}")) + }) + .collect() +} + fn set_json_pointer(value: &mut Value, pointer: &str, replacement: Value) -> Result<()> { let target = value.pointer_mut(pointer).ok_or_else(|| eyre::eyre!("missing JSON pointer {pointer}"))?; @@ -538,16 +549,37 @@ fn assert_live_sweep_record(adapter: &Value, production_ops_status: &str) -> Res let memory_evolution = find_by_field(suites, "/suite_id", "memory_evolution")?; let production_ops = find_by_field(suites, "/suite_id", "production_ops")?; let consolidation = find_by_field(suites, "/suite_id", "consolidation")?; + let knowledge = find_by_field(suites, "/suite_id", "knowledge_compilation")?; + let operator_debug = find_by_field(suites, "/suite_id", "operator_debugging_ux")?; + let capture = find_by_field(suites, "/suite_id", "capture_integration")?; + let personalization = find_by_field(suites, "/suite_id", "personalization")?; + let trust_sot = find_by_field(suites, "/suite_id", "trust_source_of_truth")?; + let retrieval = find_by_field(suites, "/suite_id", "retrieval")?; + let project_decisions = find_by_field(suites, "/suite_id", "project_decisions")?; + assert_eq!(suites.len(), 11); assert_eq!(targeted.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(full_pass.pointer("/status").and_then(Value::as_str), Some("wrong_result")); + assert!( + adapter + .pointer("/result/evidence") + .and_then(Value::as_str) + .is_some_and(|evidence| evidence.contains("38 jobs across all 11 encoded suites")) + ); + assert_eq!(trust_sot.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(work_resume.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(retrieval.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(project_decisions.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(memory_evolution.pointer("/status").and_then(Value::as_str), Some("wrong_result")); assert_eq!( production_ops.pointer("/status").and_then(Value::as_str), Some(production_ops_status) ); assert_eq!(consolidation.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(knowledge.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(operator_debug.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(capture.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(personalization.pointer("/status").and_then(Value::as_str), Some("pass")); Ok(()) } @@ -955,8 +987,12 @@ fn assert_strength_profile_summary(report: &Value) { Some("not_tested") ); assert_eq!( - report.pointer("/summary/openviking/overall_against_strengths").and_then(Value::as_str), - Some("not_tested_on_context_trajectory") + report.pointer("/summary/qmd/overall_outcome").and_then(Value::as_str), + Some("elf_loss") + ); + assert_eq!( + report.pointer("/summary/openviking/overall_outcome").and_then(Value::as_str), + Some("not_tested") ); assert_eq!( report @@ -997,11 +1033,46 @@ fn assert_strength_profile_summary(report: &Value) { fn assert_strength_profile_terms(report: &Value) -> Result<()> { let result_terms = array_at(report, "/result_type_terms")?; let coverage_terms = array_at(report, "/coverage_status_terms")?; - + let outcome_terms = array_at(report, "/outcome_terms")?; + let actual_result_terms = string_array_at(report, "/result_type_terms")?; + let actual_coverage_terms = string_array_at(report, "/coverage_status_terms")?; + + assert_eq!( + actual_result_terms, + [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported_claim", + ] + .map(str::to_owned) + ); + assert_eq!( + actual_coverage_terms, + [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported", + "unsupported_claim", + ] + .map(str::to_owned) + ); assert!(!result_terms.iter().any(|term| term.as_str() == Some("unsupported"))); + assert!(!result_terms.iter().any(|term| term.as_str() == Some("partial"))); + assert!(!coverage_terms.iter().any(|term| term.as_str() == Some("partial"))); assert!(result_terms.iter().any(|term| term.as_str() == Some("unsupported_claim"))); assert!(coverage_terms.iter().any(|term| term.as_str() == Some("unsupported"))); + assert_value_in_terms(report, "/summary/qmd/overall_outcome", outcome_terms)?; + assert_value_in_terms(report, "/summary/openviking/overall_outcome", outcome_terms)?; + for scenario in array_at(report, "/qmd_strength_profile/scenario_outcomes")? { assert_value_in_terms(scenario, "/result_type", result_terms)?; assert_value_in_terms(scenario, "/elf_status", coverage_terms)?; diff --git a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md index b74dbaf1..ef262cbc 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md +++ b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md @@ -244,8 +244,9 @@ These are needed for broad credibility but should not block personal production Do not claim: -- ELF beats qmd overall. Current live sweep is essentially tied, and qmd still owns - stronger local retrieval-debug ergonomics. +- ELF beats qmd overall. ELF is one pass ahead in the fresh aggregate because qmd + misses the delete/TTL tombstone job, but neither adapter has full-suite live pass + evidence and qmd still owns stronger local retrieval-debug ergonomics. - ELF has full-suite live real-world pass evidence. It does not. - ELF has private-corpus production quality proof. The private profile currently fails closed without an operator-owned manifest. diff --git a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json index 70b39131..91125c5f 100644 --- a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json +++ b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json @@ -44,7 +44,8 @@ ], "summary": { "qmd": { - "overall_against_strengths": "elf_loss_on_query_transparency_replay_untested", + "overall_outcome": "elf_loss", + "overall_rationale": "ELF loses the measured qmd query-transparency surface while replayability remains not_tested.", "retrieval_quality": "tie", "local_query_transparency": "elf_loss", "local_replayability": "not_tested", @@ -52,7 +53,8 @@ "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior, loses the currently evidenced local query-transparency surface, and remains untested on scored replayability, expansion, fusion, and rerank controls." }, "openviking": { - "overall_against_strengths": "not_tested_on_context_trajectory", + "overall_outcome": "not_tested", + "overall_rationale": "OpenViking context-trajectory strengths remain not_tested; ELF has only one same-corpus retrieval precondition win.", "claim": "ELF has one measured win on the same-corpus evidence-bearing precondition where OpenViking currently returns wrong_result. ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths because staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." } }, From f8f357f51c2f59deb0b6fd761c53de8bb95f1db4 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:27:30 +0800 Subject: [PATCH 07/10] {"schema":"decodex/commit/1","summary":"Repair strength-profile not-tested boundaries","authority":"XY-899"} --- .../tests/real_world_job_benchmark.rs | 79 ++++++++++++++++--- ...-qmd-openviking-strength-profile-report.md | 21 ++--- ...md-openviking-strength-profile-report.json | 20 ++--- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 8c8b18b7..8be178c7 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -115,6 +115,22 @@ fn competitor_strength_matrix_path() -> Result { .join("2026-06-11-competitor-strength-evidence-matrix.md")) } +fn readme_path() -> Result { + Ok(workspace_root()?.join("README.md")) +} + +fn benchmarking_index_path() -> Result { + Ok(workspace_root()?.join("docs").join("guide").join("benchmarking").join("index.md")) +} + +fn iteration_direction_report_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md")) +} + fn external_adapter_manifest_path() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .join("fixtures") @@ -872,6 +888,9 @@ fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result let report = serde_json::from_str::(&fs::read_to_string(strength_profile_report_path()?)?)?; let markdown = fs::read_to_string(strength_profile_markdown_path()?)?; + let readme = fs::read_to_string(readme_path()?)?; + let benchmarking_index = fs::read_to_string(benchmarking_index_path()?)?; + let iteration_direction = fs::read_to_string(iteration_direction_report_path()?)?; assert_strength_profile_summary(&report); assert_strength_profile_terms(&report)?; @@ -880,6 +899,11 @@ fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result assert_openviking_strength_profile(&report)?; assert_strength_profile_json_claim_boundaries(&report)?; assert_strength_profile_markdown_boundaries(&markdown); + assert_operator_facing_strength_profile_boundaries( + &readme, + &benchmarking_index, + &iteration_direction, + ); Ok(()) } @@ -980,7 +1004,7 @@ fn assert_strength_profile_summary(report: &Value) { ); assert_eq!( report.pointer("/summary/qmd/local_query_transparency").and_then(Value::as_str), - Some("elf_loss") + Some("not_tested") ); assert_eq!( report.pointer("/summary/qmd/local_replayability").and_then(Value::as_str), @@ -988,7 +1012,7 @@ fn assert_strength_profile_summary(report: &Value) { ); assert_eq!( report.pointer("/summary/qmd/overall_outcome").and_then(Value::as_str), - Some("elf_loss") + Some("not_tested") ); assert_eq!( report.pointer("/summary/openviking/overall_outcome").and_then(Value::as_str), @@ -1008,13 +1032,13 @@ fn assert_strength_profile_summary(report: &Value) { report .pointer("/qmd_strength_profile/win_tie_loss_summary/elf_loss") .and_then(Value::as_u64), - Some(1) + Some(0) ); assert_eq!( report .pointer("/qmd_strength_profile/win_tie_loss_summary/not_tested") .and_then(Value::as_u64), - Some(4) + Some(5) ); assert_eq!( report @@ -1115,7 +1139,11 @@ fn assert_qmd_strength_profile(report: &Value) -> Result<()> { assert_eq!(retrieval.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); assert_eq!( local_transparency.pointer("/elf_outcome").and_then(Value::as_str), - Some("elf_loss") + Some("not_tested") + ); + assert_eq!( + local_transparency.pointer("/result_type").and_then(Value::as_str), + Some("not_encoded") ); assert_eq!( rerank_controls.pointer("/result_type").and_then(Value::as_str), @@ -1205,7 +1233,7 @@ fn assert_strength_profile_json_claim_boundaries(report: &Value) -> Result<()> { assert!(array_contains_str( report, "/claim_boundaries", - "ELF does not broadly beat qmd; it ties retrieval correctness, loses the measured query-transparency surface, and leaves replayability not_tested." + "ELF does not broadly beat qmd; it ties encoded retrieval and lifecycle correctness, keeps qmd query transparency as not_tested for comparative scoring, and leaves replayability not_tested." )?); assert!(array_contains_str( report, @@ -1240,10 +1268,8 @@ fn assert_strength_profile_markdown_boundaries(markdown: &str) { assert!( markdown.contains("ELF ties qmd on the current encoded retrieval-correctness surfaces") ); - assert!(markdown.contains( - "qmd remains stronger than ELF on the currently evidenced local query transparency" - )); - assert!(markdown.contains("replayability remains unscored")); + assert!(markdown.contains("qmd remains the local retrieval-debug UX reference")); + assert!(markdown.contains("not scored as comparative ELF wins or losses")); assert!(markdown.contains("ELF currently wins only the equivalent OpenViking same-corpus")); assert!(markdown.contains("Do not claim ELF broadly beats qmd")); assert!(markdown.contains( @@ -1256,6 +1282,39 @@ fn assert_strength_profile_markdown_boundaries(markdown: &str) { assert!(markdown.contains("typed `wrong_result` state")); } +fn assert_operator_facing_strength_profile_boundaries( + readme: &str, + benchmarking_index: &str, + iteration_direction: &str, +) { + assert!(readme.contains("Full-suite live real-world adapter sweep after XY-899")); + assert!(readme.contains("qmd remains the local retrieval-debug UX reference")); + assert!(readme.contains("no broad ELF-over-qmd claim is allowed")); + assert!(readme.contains("qmd and OpenViking Strength-Profile Report - June 11, 2026")); + assert!(benchmarking_index.contains("2026-06-11-qmd-openviking-strength-profile-report.md")); + assert!( + benchmarking_index.contains("separates qmd retrieval quality from debug/replay ergonomics") + ); + assert!(benchmarking_index.contains("preserves OpenViking context-trajectory")); + assert!( + benchmarking_index + .contains("surfaces as `not_tested` until staged/hierarchical evidence is encoded") + ); + assert!( + iteration_direction + .contains("ELF and qmd are tied on the encoded live retrieval, work-resume, and") + ); + assert!(iteration_direction.contains("ELF does not yet beat qmd's local retrieval-debug")); + assert!( + iteration_direction + .contains("ELF beats OpenViking on context trajectory. That scenario is not encoded.") + ); + assert!( + iteration_direction + .contains("Do not promote a reference project into a win/loss claim until") + ); +} + #[test] fn generated_json_report_renders_markdown() -> Result<()> { let report = run_json_report()?; diff --git a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md index 37c9c0e6..99b1260a 100644 --- a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md +++ b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md @@ -21,10 +21,11 @@ The measured qmd judgment is narrower: - Retrieval quality: `tie`. ELF and qmd both pass the encoded live real-world retrieval suite and both pass the 480-document stress retrieval baseline. -- Local query transparency: `elf_loss`. qmd's current artifacts expose directly +- Local query transparency: `not_tested`. qmd's current artifacts expose directly inspectable top-10 JSON rows with files, line numbers, snippets, and scores. ELF has stronger service traces and production-operation evidence, but the checked-in - stress report does not hydrate an equivalent candidate list. + stress report does not hydrate an equivalent candidate list, so no scored ELF loss + is claimed for this surface. - Local replayability: `not_tested`. qmd has a concise observed CLI replay path, and ELF has service traces plus admin bundle endpoints, but no scored replayability rule compares those surfaces yet. @@ -48,7 +49,7 @@ The measured OpenViking judgment is split by surface: | Scenario | Evidence Class | Result Type | ELF Outcome | What It Means | | --- | --- | --- | --- | --- | | Retrieval quality | `live_real_world` | `pass` | `tie` | Both systems pass 5/5 live retrieval jobs with 6/6 expected evidence matched. | -| Local query transparency | `live_baseline_only` | `pass` | `elf_loss` | qmd exposes top-10 files, line numbers, snippets, scores, and distractor density directly in the stress artifact. | +| Local query transparency | `live_baseline_only` | `not_encoded` | `not_tested` | qmd exposes top-10 files, line numbers, snippets, scores, and distractor density directly in the stress artifact, but the equivalent ELF candidate-list surface is not encoded. | | Expansion/fusion/rerank controls | `research_gate` | `not_encoded` | `not_tested` | No scored profile proves either system's expansion, fusion, or rerank superiority. | | Stale context isolation | `live_real_world` | `pass` | `tie` | Both systems pass the encoded current-vs-obsolete and distractor-heavy retrieval jobs. | | Update/delete/cold-start behavior | `live_baseline_only` | `pass` | `tie` | Equivalent update replacement, delete suppression, and cold-start recovery checks pass for both. | @@ -56,10 +57,11 @@ The measured OpenViking judgment is split by surface: | Local replayability | `live_baseline_only` | `not_encoded` | `not_tested` | qmd has a shorter observed CLI replay path, but no scored replayability rule compares it with ELF's trace/admin replay surfaces yet. | | Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` | The report classifies qmd memory-evolution failures, but qmd candidate-drop traces are not yet materialized and no pass evidence is claimed. | -Summary: qmd strength-profile outcomes are `0` ELF wins, `3` ties, `1` ELF loss, -and `4` not-tested scenarios. This distinguishes retrieval quality from -debug/replay ergonomics: the retrieval result is tied, the checked-in query-debug -artifact ergonomics currently favor qmd, and replayability remains unscored. +Summary: qmd strength-profile outcomes are `0` ELF wins, `3` ties, `0` ELF losses, +and `5` not-tested scenarios. This distinguishes retrieval quality from +debug/replay ergonomics: the retrieval result is tied, qmd remains the local +retrieval-debug UX reference, and query transparency plus replayability remain +unscored for comparative ELF win/loss claims. ## qmd Wrong-Result Diagnosis @@ -99,8 +101,9 @@ context-trajectory strengths remain not tested. Allowed: - ELF ties qmd on the current encoded retrieval-correctness surfaces. -- qmd remains stronger than ELF on the currently evidenced local query transparency - artifact ergonomics; replayability is observed but not scored. +- qmd remains the local retrieval-debug UX reference on the currently evidenced query + transparency artifact ergonomics; query transparency and replayability are observed + but not scored as comparative ELF wins or losses. - qmd expansion/fusion/rerank superiority is untested. - OpenViking's Docker local embedding setup reaches runtime, but context trajectory remains untested because evidence-bearing same-corpus retrieval is not passing. diff --git a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json index 91125c5f..d8d966d6 100644 --- a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json +++ b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json @@ -44,13 +44,13 @@ ], "summary": { "qmd": { - "overall_outcome": "elf_loss", - "overall_rationale": "ELF loses the measured qmd query-transparency surface while replayability remains not_tested.", + "overall_outcome": "not_tested", + "overall_rationale": "ELF ties qmd on encoded retrieval and lifecycle surfaces; qmd query-transparency, replayability, and expansion/fusion/rerank strengths remain not_tested for comparative scoring because equivalent scored ELF surfaces are not encoded.", "retrieval_quality": "tie", - "local_query_transparency": "elf_loss", + "local_query_transparency": "not_tested", "local_replayability": "not_tested", "expansion_fusion_rerank": "not_tested", - "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior, loses the currently evidenced local query-transparency surface, and remains untested on scored replayability, expansion, fusion, and rerank controls." + "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior. qmd remains the local retrieval-debug UX reference, but ELF has no scored loss on query-transparency, replayability, expansion, fusion, or rerank controls until equivalent comparative surfaces are encoded." }, "openviking": { "overall_outcome": "not_tested", @@ -80,12 +80,12 @@ "scenario_id": "qmd-local-query-transparency", "surface": "local query transparency", "evidence_class": "live_baseline_only", - "result_type": "pass", + "result_type": "not_encoded", "elf_status": "not_encoded", "qmd_status": "pass", - "elf_outcome": "elf_loss", + "elf_outcome": "not_tested", "retrieval_quality": "not a correctness scenario", - "debug_replay_ergonomics": "qmd stress artifacts expose per-query top-10 files, line numbers, snippets, scores, and distractor density; ELF stress artifacts expose trace ids and top evidence but do not hydrate the candidate list in the checked-in report.", + "debug_replay_ergonomics": "qmd stress artifacts expose per-query top-10 files, line numbers, snippets, scores, and distractor density; ELF stress artifacts expose trace ids and top evidence but do not hydrate an equivalent candidate list in the checked-in report, so this surface is not scored as a comparative ELF loss.", "source_artifacts": [ "scripts/live-baseline-benchmark.sh", "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" @@ -186,8 +186,8 @@ "win_tie_loss_summary": { "elf_win": 0, "tie": 3, - "elf_loss": 1, - "not_tested": 4 + "elf_loss": 0, + "not_tested": 5 }, "wrong_result_diagnosis": { "taxonomy": [ @@ -367,7 +367,7 @@ } }, "claim_boundaries": [ - "ELF does not broadly beat qmd; it ties retrieval correctness, loses the measured query-transparency surface, and leaves replayability not_tested.", + "ELF does not broadly beat qmd; it ties encoded retrieval and lifecycle correctness, keeps qmd query transparency as not_tested for comparative scoring, and leaves replayability not_tested.", "qmd expansion, fusion, and rerank superiority remains not_tested because the current qmd paths use --no-rerank and do not score internals.", "ELF does not beat OpenViking on context trajectory; OpenViking trajectory strengths remain not_tested behind a wrong_result same-corpus output precondition.", "Research_gate records are follow-up gates, not pass evidence.", From c6ffc5f49a3fc8950b76f9fe740108a0c4582beb Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:38:12 +0800 Subject: [PATCH 08/10] {"schema":"decodex/commit/1","summary":"Repair adapter manifest boundary wording","authority":"XY-899"} --- .../memory_projects_manifest.json | 6 ++-- .../tests/real_world_job_benchmark.rs | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 4b0cb84e..89ef31cc 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -290,7 +290,7 @@ }, "result": { "status": "pass", - "evidence": "The current evidence is same-corpus live-baseline evidence only; no real_world_job qmd adapter is encoded yet.", + "evidence": "This live_baseline_only record is same-corpus evidence only; cite qmd_live_real_world for the full live real-world sweep.", "artifact": "docs/guide/benchmarking/live_baseline_benchmark.md" }, "capabilities": [ @@ -314,7 +314,7 @@ { "suite_id": "retrieval", "status": "not_encoded", - "evidence": "qmd is a retrieval-debug reference, but no real_world_job retrieval adapter run is encoded." + "evidence": "This live_baseline_only record does not execute real_world_job retrieval prompts; cite qmd_live_real_world for the live retrieval adapter run." }, { "suite_id": "memory_evolution", @@ -1018,7 +1018,7 @@ { "capability": "hierarchical_context_trajectory", "status": "not_encoded", - "evidence": "Stage trajectory scoring is not encoded until setup reaches runnable OpenViking APIs." + "evidence": "Stage trajectory scoring remains not encoded until the smoke adapter returns evidence-bearing same-corpus output instead of the current wrong_result missed-term evidence." }, { "capability": "host_global_install_boundary", diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 8be178c7..f1b555d5 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -405,6 +405,9 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { assert_eq!(qmd.pointer("/overall_status").and_then(Value::as_str), Some("pass")); assert_eq!(qmd.pointer("/suites/0/status").and_then(Value::as_str), Some("not_encoded")); + + assert_qmd_live_baseline_record(qmd); + assert_eq!( qmd_live.pointer("/evidence_class").and_then(Value::as_str), Some("live_real_world") @@ -477,6 +480,9 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { openviking_deep.pointer("/adapter_kind").and_then(Value::as_str), Some("docker_local_embed_context_trajectory_gate") ); + + assert_openviking_deep_profile_gate(openviking_deep); + assert_eq!( openviking_deep.pointer("/result/artifact").and_then(Value::as_str), Some("docs/research/2026-06-11-qmd-openviking-strength-profile-report.json") @@ -485,6 +491,32 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { Ok(()) } +fn assert_qmd_live_baseline_record(adapter: &Value) { + let result_evidence = adapter.pointer("/result/evidence").and_then(Value::as_str); + let retrieval_evidence = adapter.pointer("/suites/0/evidence").and_then(Value::as_str); + + assert!(result_evidence.is_some_and(|evidence| { + evidence.contains("This live_baseline_only record is same-corpus evidence only") + && evidence.contains("cite qmd_live_real_world for the full live real-world sweep") + && !evidence.contains("no real_world_job qmd adapter is encoded yet") + })); + assert!(retrieval_evidence.is_some_and(|evidence| { + evidence.contains("does not execute real_world_job retrieval prompts") + && evidence.contains("cite qmd_live_real_world for the live retrieval adapter run") + && !evidence.contains("no real_world_job retrieval adapter run is encoded") + })); +} + +fn assert_openviking_deep_profile_gate(adapter: &Value) { + let trajectory_evidence = adapter.pointer("/capabilities/1/evidence").and_then(Value::as_str); + + assert!(trajectory_evidence.is_some_and(|evidence| { + evidence.contains("evidence-bearing same-corpus output") + && evidence.contains("wrong_result missed-term evidence") + && !evidence.contains("setup reaches runnable OpenViking APIs") + })); +} + fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, claude_mem: &Value) { assert_eq!( mem0.pointer("/capabilities/2/capability").and_then(Value::as_str), From 9a02f6b11f354dd395bfb1a96b410a3bffc5b363 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:51:27 +0800 Subject: [PATCH 09/10] {"schema":"decodex/commit/1","summary":"Add matrix and README claim guards","authority":"XY-899"} --- .../tests/real_world_job_benchmark.rs | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index f1b555d5..02bfc34b 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -115,6 +115,13 @@ fn competitor_strength_matrix_path() -> Result { .join("2026-06-11-competitor-strength-evidence-matrix.md")) } +fn competitor_strength_matrix_json_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-xy-897-competitor-strength-matrix.json")) +} + fn readme_path() -> Result { Ok(workspace_root()?.join("README.md")) } @@ -947,6 +954,9 @@ fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { measurement_coverage_audit_json_path()?, )?)?; let competitor_matrix = fs::read_to_string(competitor_strength_matrix_path()?)?; + let competitor_matrix_json = serde_json::from_str::(&fs::read_to_string( + competitor_strength_matrix_json_path()?, + )?)?; let external_manifest = fs::read_to_string(external_adapter_manifest_path())?; let retrieval_debug_profile = serde_json::from_str::(&fs::read_to_string(retrieval_debug_profile_json_path()?)?)?; @@ -1022,6 +1032,84 @@ fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { Some(6) ); + assert_competitor_strength_matrix_json(&competitor_matrix_json)?; + + Ok(()) +} + +fn assert_competitor_strength_matrix_json(matrix: &Value) -> Result<()> { + let projects = array_at(matrix, "/project_matrix")?; + let qmd = find_by_field(projects, "/project", "qmd")?; + let openviking = find_by_field(projects, "/project", "OpenViking")?; + + assert_eq!( + qmd.pointer("/current_evidence_class").and_then(Value::as_str), + Some("live_real_world") + ); + assert_eq!(qmd.pointer("/measured_status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + qmd.pointer("/unsupported_or_blocked_status/state").and_then(Value::as_str), + Some("not_encoded") + ); + assert!(qmd.pointer("/benchmark_before_claim").and_then(Value::as_str).is_some_and(|claim| { + claim.contains("before claiming ELF wins, ties, or loses on retrieval debugging") + })); + assert!( + qmd.pointer("/borrow_if_stronger") + .and_then(Value::as_str) + .is_some_and(|claim| claim.contains("transparent local knobs")) + ); + assert_eq!( + openviking.pointer("/current_evidence_class").and_then(Value::as_str), + Some("live_baseline_only") + ); + assert_eq!( + openviking.pointer("/measured_status").and_then(Value::as_str), + Some("wrong_result") + ); + assert_eq!( + openviking.pointer("/unsupported_or_blocked_status/state").and_then(Value::as_str), + Some("not_encoded") + ); + assert!( + openviking + .pointer("/unsupported_or_blocked_status/details") + .and_then(Value::as_str) + .is_some_and(|details| details.contains("same-corpus output misses expected evidence")) + ); + assert!( + openviking + .pointer("/benchmark_before_claim") + .and_then(Value::as_str) + .is_some_and(|claim| claim.contains("evidence-bearing same-corpus output pass")) + ); + + let scenarios = array_at(matrix, "/scenario_matrix")?; + let retrieval_debug = find_by_field(scenarios, "/scenario_id", "retrieval_debug")?; + let context_trajectory = find_by_field(scenarios, "/scenario_id", "context_trajectory")?; + + assert!( + retrieval_debug + .pointer("/current_state") + .and_then(Value::as_str) + .is_some_and(|state| state.contains("Measured tie on encoded retrieval answers")) + ); + assert!(retrieval_debug.pointer("/current_state").and_then(Value::as_str).is_some_and( + |state| state.contains("qmd remains stronger on local debug ergonomics not fully scored") + )); + assert!( + context_trajectory + .pointer("/current_state") + .and_then(Value::as_str) + .is_some_and(|state| state.contains("not a measured live winner")) + ); + assert!( + context_trajectory + .pointer("/next_measurement") + .and_then(Value::as_str) + .is_some_and(|measurement| measurement.contains("evidence-bearing retrieval pass")) + ); + Ok(()) } @@ -1320,6 +1408,12 @@ fn assert_operator_facing_strength_profile_boundaries( iteration_direction: &str, ) { assert!(readme.contains("Full-suite live real-world adapter sweep after XY-899")); + assert!(readme.contains("fresh ELF sweep reports 18 pass")); + assert!(readme.contains("5 wrong_result, 2 blocked, and 13 not_encoded jobs")); + assert!(readme.contains("fresh qmd sweep reports")); + assert!(readme.contains("17 pass, 6 wrong_result, 2 blocked, and 13 not_encoded jobs")); + assert!(readme.contains("The difference is the")); + assert!(readme.contains("delete/TTL tombstone case")); assert!(readme.contains("qmd remains the local retrieval-debug UX reference")); assert!(readme.contains("no broad ELF-over-qmd claim is allowed")); assert!(readme.contains("qmd and OpenViking Strength-Profile Report - June 11, 2026")); From 154bdbfd9832d0b2c9ae59604ee9ca149e9055fd Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 15:03:04 +0800 Subject: [PATCH 10/10] {"schema":"decodex/commit/1","summary":"Align external adapter project counts","authority":"XY-899"} --- .../src/bin/real_world_job_benchmark.rs | 7 ++- .../tests/real_world_job_benchmark.rs | 49 ++++++++++++++++++- .../2026-06-11-measurement-coverage-audit.md | 13 +++-- ...2026-06-11-measurement-coverage-audit.json | 2 +- .../real_world_agent_memory_benchmark_v1.md | 6 +++ 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark.rs b/apps/elf-eval/src/bin/real_world_job_benchmark.rs index e987986b..b0d31fa1 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark.rs @@ -3882,9 +3882,14 @@ fn validate_adapter_execution_metadata(path: &Path, adapter: &ExternalAdapterRep } fn external_adapter_summary(adapters: &[ExternalAdapterReport]) -> ExternalAdapterSummary { + let external_projects = adapters + .iter() + .filter(|adapter| adapter.project != "ELF") + .map(|adapter| adapter.project.as_str()) + .collect::>(); let mut summary = ExternalAdapterSummary { adapter_count: adapters.len(), - external_project_count: adapters.iter().filter(|adapter| adapter.project != "ELF").count(), + external_project_count: external_projects.len(), ..ExternalAdapterSummary::default() }; diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 02bfc34b..ac98cc9d 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -305,7 +305,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { ); assert_eq!( report.pointer("/external_adapters/summary/external_project_count").and_then(Value::as_u64), - Some(19) + Some(16) ); assert_eq!( report.pointer("/external_adapters/summary/fixture_backed_count").and_then(Value::as_u64), @@ -1040,6 +1040,7 @@ fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { fn assert_competitor_strength_matrix_json(matrix: &Value) -> Result<()> { let projects = array_at(matrix, "/project_matrix")?; let qmd = find_by_field(projects, "/project", "qmd")?; + let mem0 = find_by_field(projects, "/project", "mem0/OpenMemory")?; let openviking = find_by_field(projects, "/project", "OpenViking")?; assert_eq!( @@ -1059,6 +1060,16 @@ fn assert_competitor_strength_matrix_json(matrix: &Value) -> Result<()> { .and_then(Value::as_str) .is_some_and(|claim| claim.contains("transparent local knobs")) ); + assert_eq!(mem0.pointer("/measured_status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + mem0.pointer("/unsupported_or_blocked_status/state").and_then(Value::as_str), + Some("not_encoded") + ); + assert!( + mem0.pointer("/benchmark_before_claim") + .and_then(Value::as_str) + .is_some_and(|claim| claim.contains("Fix the local adapter's same-corpus result")) + ); assert_eq!( openviking.pointer("/current_evidence_class").and_then(Value::as_str), Some("live_baseline_only") @@ -1252,6 +1263,11 @@ fn assert_qmd_strength_profile(report: &Value) -> Result<()> { let retrieval = find_by_field(qmd_scenarios, "/scenario_id", "qmd-retrieval-quality")?; let rerank_controls = find_by_field(qmd_scenarios, "/scenario_id", "qmd-expansion-fusion-rerank-controls")?; + let stale_isolation = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-stale-context-isolation")?; + let lifecycle = find_by_field(qmd_scenarios, "/scenario_id", "qmd-update-delete-cold-start")?; + let operator_debug = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-operator-debug-evidence")?; let replayability = find_by_field(qmd_scenarios, "/scenario_id", "qmd-local-replayability")?; let wrong_result = find_by_field(qmd_scenarios, "/scenario_id", "qmd-wrong-result-diagnosis")?; @@ -1269,6 +1285,12 @@ fn assert_qmd_strength_profile(report: &Value) -> Result<()> { rerank_controls.pointer("/result_type").and_then(Value::as_str), Some("not_encoded") ); + assert_eq!(stale_isolation.pointer("/result_type").and_then(Value::as_str), Some("pass")); + assert_eq!(stale_isolation.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); + assert_eq!(lifecycle.pointer("/result_type").and_then(Value::as_str), Some("pass")); + assert_eq!(lifecycle.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); + assert_eq!(operator_debug.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(operator_debug.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); assert_eq!(replayability.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); assert_eq!(replayability.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); assert_eq!( @@ -1325,11 +1347,20 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { "/scenario_id", "openviking-evidence-bearing-retrieval-precondition", )?; + let local_embed_setup = + find_by_field(openviking_scenarios, "/scenario_id", "openviking-local-embed-setup")?; let missed_terms = find_by_field( openviking_scenarios, "/scenario_id", "openviking-missed-expected-terms-evidence", )?; + let hierarchy = + find_by_field(openviking_scenarios, "/scenario_id", "openviking-hierarchy-selection")?; + let recursive_expansion = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-recursive-context-expansion", + )?; assert_eq!(openviking_scenarios.len(), 6); assert_eq!( @@ -1337,6 +1368,12 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { Some("research_gate") ); assert_eq!(trajectory.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(local_embed_setup.pointer("/result_type").and_then(Value::as_str), Some("pass")); + assert_eq!( + local_embed_setup.pointer("/elf_outcome").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!(local_embed_setup.pointer("/typed_blocker"), Some(&Value::Null)); assert_eq!(precondition.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); assert_eq!(precondition.pointer("/elf_outcome").and_then(Value::as_str), Some("elf_win")); assert_eq!( @@ -1345,6 +1382,16 @@ fn assert_openviking_strength_profile(report: &Value) -> Result<()> { ); assert_eq!(missed_terms.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); assert_eq!(missed_terms.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); + assert_eq!(hierarchy.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(hierarchy.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); + assert_eq!( + recursive_expansion.pointer("/result_type").and_then(Value::as_str), + Some("not_encoded") + ); + assert_eq!( + recursive_expansion.pointer("/elf_outcome").and_then(Value::as_str), + Some("not_tested") + ); Ok(()) } diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index 4c1a9657..6ca69c9a 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -133,10 +133,9 @@ The checked-in manifest records 21 adapter records across 17 unique project name | `blocked` | `6` | | `not_encoded` | `7` | -The generated JSON report also emits `external_project_count: 19`, while the unique -project-name count from the manifest is 17. The runner currently computes that field -as adapter records whose project is not `ELF`, not as unique external project names. -Interpret the unique manifest project list as the project coverage count. +The generated JSON report emits `external_project_count: 16`, matching the unique +non-ELF project-name count from the manifest. The companion audit JSON separately +records `unique_project_names: 17` for the full project list including ELF. ## Project Coverage @@ -218,9 +217,9 @@ Order these by decision value, not implementation convenience: - Output: Docker-contained artifacts mapped to evidence ids, or typed setup and resource blockers. -Before publishing the next aggregate report, clarify or rename the generated -`external_project_count` field so readers do not confuse non-ELF adapter records with -unique external projects. +Before publishing the next aggregate report, keep `external_project_count` aligned +with unique non-ELF project names so readers do not confuse project coverage with +adapter-record coverage. ## Fail Criteria diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index 575bdf6b..5fb3447f 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -83,7 +83,7 @@ "adapter_ledger": { "adapter_records": 21, "unique_project_names": 17, - "external_project_count_note": "The generated report field external_project_count currently counts non-ELF adapter records, not unique external project names.", + "external_project_count_note": "The generated report field external_project_count counts unique non-ELF project names, not adapter records.", "evidence_class_counts": { "fixture_backed": 1, "live_baseline_only": 6, diff --git a/docs/spec/real_world_agent_memory_benchmark_v1.md b/docs/spec/real_world_agent_memory_benchmark_v1.md index bb0a4b82..f6e1470d 100644 --- a/docs/spec/real_world_agent_memory_benchmark_v1.md +++ b/docs/spec/real_world_agent_memory_benchmark_v1.md @@ -228,6 +228,12 @@ metadata, per-adapter records, and summary counters for: - capability coverage statuses; - real-world suite coverage statuses. +For `elf.real_world_external_adapter_report/v1`, `adapter_count` is the number of +adapter records in the loaded manifest. `external_project_count` is the number of +unique non-ELF project names represented by those records, not the number of non-ELF +adapter records. Multiple adapter records for the same external project MUST count as +one external project in this summary. + Adapter-pack issues SHOULD add new projects by appending adapter records to this manifest shape. They MUST NOT change these status meanings to make a project look better or worse.