diff --git a/README.md b/README.md index 550c3587..bdd884b3 100644 --- a/README.md +++ b/README.md @@ -153,12 +153,15 @@ provider-backed ELF evidence was required. jobs across 11 suites, 36 pass, 0 incomplete, 2 blocked, 0 wrong-result, 0 not-encoded, and 0 unsupported-claim results. The remaining non-pass jobs are production-ops operator boundaries, not hidden benchmark wins. -- Full-suite live real-world adapter sweep after XY-880: ELF and qmd now emit +- Full-suite live real-world adapter sweep after XY-899: ELF and qmd emit Docker-isolated `live_real_world` records for all 38 encoded jobs across 11 suites through `cargo make real-world-memory-live-adapters`. Both keep the original targeted `work_resume`, `retrieval`, and `project_decisions` slice passing, but the - full sweep is not a full-suite pass: each adapter reports 18 pass, 5 wrong_result, - 1 incomplete, 2 blocked, and 12 not_encoded jobs. + full sweep is not a full-suite pass. The fresh ELF sweep reports 18 pass, + 5 wrong_result, 2 blocked, and 13 not_encoded jobs. The fresh qmd sweep reports + 17 pass, 6 wrong_result, 2 blocked, and 13 not_encoded jobs. The difference is the + delete/TTL tombstone case; qmd remains the local retrieval-debug UX reference, and + no broad ELF-over-qmd claim is allowed. - Expanded adapter-pack coverage after XY-834: the real-world external adapter manifest now includes `research_gate` records for RAGFlow, LightRAG, GraphRAG, Graphiti/Zep, Letta, LangGraph, nanograph, llm-wiki, gbrain, and deeper @@ -191,6 +194,7 @@ Detailed evidence and interpretation: - [Real-World Comparison Report - June 10, 2026](docs/guide/benchmarking/2026-06-10-real-world-comparison-report.md) - [Live Real-World Adapter Sweep Report - June 10, 2026](docs/guide/benchmarking/2026-06-10-live-real-world-sweep-report.md) - [Post-Adapter Production Adoption Refresh - June 10, 2026](docs/guide/benchmarking/2026-06-10-production-adoption-refresh.md) +- [qmd and OpenViking Strength-Profile Report - June 11, 2026](docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md) - [Graph/RAG Scored Smoke Adapter Report - June 11, 2026](docs/guide/benchmarking/2026-06-11-graph-rag-scored-smoke-adapter-report.md) - [Live Baseline Benchmark Runbook](docs/guide/benchmarking/live_baseline_benchmark.md) - [Single-User Production Runbook](docs/guide/single_user_production.md) @@ -204,7 +208,7 @@ Detailed evidence and interpretation: live sweep, but that sweep still contains typed non-pass states and is not full-suite parity. -Evidence-backed position after the June 10 real-world report: +Evidence-backed position after the June 11 real-world reports: - ELF is better evidenced than the tested alternatives on evidence-bound writes, deterministic ingestion boundaries, Postgres source-of-truth plus rebuildable Qdrant @@ -276,7 +280,7 @@ Detailed comparison, mechanism-level analysis, and source map: - [RAG/Graph Adapter Feasibility Research Run](docs/research/2026-06-10-xy-882-rag-graph-adapter-feasibility.json) Latest real-world benchmark report: June 11, 2026. Latest external research refresh: -June 10, 2026. +June 11, 2026. ## Documentation diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 3ce54053..3c023fe2 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -290,7 +290,7 @@ }, "result": { "status": "pass", - "evidence": "The current evidence is same-corpus live-baseline evidence only; no real_world_job qmd adapter is encoded yet.", + "evidence": "This live_baseline_only record is same-corpus evidence only; cite qmd_live_real_world for the full live real-world sweep.", "artifact": "docs/guide/benchmarking/live_baseline_benchmark.md" }, "capabilities": [ @@ -314,7 +314,7 @@ { "suite_id": "retrieval", "status": "not_encoded", - "evidence": "qmd is a retrieval-debug reference, but no real_world_job retrieval adapter run is encoded." + "evidence": "This live_baseline_only record does not execute real_world_job retrieval prompts; cite qmd_live_real_world for the live retrieval adapter run." }, { "suite_id": "memory_evolution", @@ -425,7 +425,7 @@ { "suite_id": "memory_evolution", "status": "wrong_result", - "evidence": "qmd passed the delete/TTL case but failed five current-versus-historical conflict jobs because retrieval-backed answers did not provide the required historical conflict evidence links." + "evidence": "qmd failed all six memory-evolution jobs in the fresh June 11 diagnostic, including the delete/TTL tombstone job where qmd retrieved only the current plan and missed the tombstone evidence." }, { "suite_id": "consolidation", @@ -1036,11 +1036,12 @@ }, "run": { "status": "not_encoded", - "evidence": "No expanded qmd stress or real_world_job deep-profile artifact is checked in for this adapter-pack gate." + "evidence": "The XY-899 strength-profile report is checked in, but no new live qmd deep-profile adapter artifact is claimed from it." }, "result": { "status": "not_encoded", - "evidence": "qmd deep retrieval-debug evidence remains a planned profile, not a new pass claim." + "evidence": "The XY-899 report records qmd scenario-level retrieval/debug/replay outcomes and wrong-result diagnosis taxonomy, while expansion/fusion/rerank scoring remains not_encoded.", + "artifact": "docs/research/2026-06-11-qmd-openviking-strength-profile-report.json" }, "capabilities": [ { @@ -1051,7 +1052,7 @@ { "capability": "real_world_job_adapter", "status": "not_encoded", - "evidence": "The qmd live real-world slice covers representative jobs only; expanded retrieval-debug suites need their own materialized adapter run." + "evidence": "The qmd live real-world sweep covers the current encoded fixture corpus; expanded retrieval-debug strength suites still need their own materialized adapter run." }, { "capability": "host_global_install_boundary", @@ -1107,7 +1108,7 @@ { "adapter_id": "openviking_deep_profile_gate", "project": "OpenViking", - "adapter_kind": "docker_local_embed_deep_profile_gate", + "adapter_kind": "docker_local_embed_context_trajectory_gate", "evidence_class": "research_gate", "docker_default": true, "host_global_installs_required": false, @@ -1120,11 +1121,12 @@ }, "run": { "status": "not_encoded", - "evidence": "The adapter cannot fairly exercise hierarchical trajectory behavior until same-corpus add_resource/find returns evidence-bearing results." + "evidence": "The XY-899 strength-profile report records staged retrieval, hierarchy selection, recursive/context expansion, and missed-term evidence as typed not_tested or wrong_result states; no new live trajectory adapter artifact is claimed." }, "result": { "status": "not_encoded", - "evidence": "No OpenViking deep context-trajectory result is claimed from the current wrong-result smoke run." + "evidence": "No OpenViking deep context-trajectory result is claimed from the current wrong-result smoke run; the XY-899 report preserves the trajectory surfaces as not_tested.", + "artifact": "docs/research/2026-06-11-qmd-openviking-strength-profile-report.json" }, "capabilities": [ { @@ -1135,7 +1137,7 @@ { "capability": "hierarchical_context_trajectory", "status": "not_encoded", - "evidence": "Stage trajectory scoring is not encoded until setup reaches runnable OpenViking APIs." + "evidence": "Stage trajectory scoring remains not encoded until the smoke adapter returns evidence-bearing same-corpus output instead of the current wrong_result missed-term evidence." }, { "capability": "host_global_install_boundary", diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 46fc4660..ce163f29 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -60,6 +60,91 @@ fn production_ops_fixture_dir() -> PathBuf { real_world_memory_fixture_dir().join("production_ops") } +fn workspace_root() -> Result { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let root = manifest_dir + .parent() + .and_then(Path::parent) + .ok_or_else(|| eyre::eyre!("could not resolve workspace root"))?; + + Ok(root.to_path_buf()) +} + +fn strength_profile_report_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-qmd-openviking-strength-profile-report.json")) +} + +fn strength_profile_markdown_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-qmd-openviking-strength-profile-report.md")) +} + +fn measurement_coverage_audit_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-measurement-coverage-audit.md")) +} + +fn measurement_coverage_audit_json_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-measurement-coverage-audit.json")) +} + +fn retrieval_debug_profile_json_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-elf-qmd-retrieval-debug-profile.json")) +} + +fn competitor_strength_matrix_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-competitor-strength-evidence-matrix.md")) +} + +fn competitor_strength_matrix_json_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("research") + .join("2026-06-11-xy-897-competitor-strength-matrix.json")) +} + +fn readme_path() -> Result { + Ok(workspace_root()?.join("README.md")) +} + +fn benchmarking_index_path() -> Result { + Ok(workspace_root()?.join("docs").join("guide").join("benchmarking").join("index.md")) +} + +fn iteration_direction_report_path() -> Result { + Ok(workspace_root()? + .join("docs") + .join("guide") + .join("benchmarking") + .join("2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md")) +} + +fn external_adapter_manifest_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join("real_world_external_adapters") + .join("memory_projects_manifest.json") +} + fn run_json_report_from(fixtures: PathBuf) -> Result { let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) .arg("run") @@ -98,6 +183,17 @@ fn array_contains_str(value: &Value, pointer: &str, expected: &str) -> Result Result> { + array_at(value, pointer)? + .iter() + .map(|item| { + item.as_str() + .map(str::to_owned) + .ok_or_else(|| eyre::eyre!("non-string entry at {pointer}")) + }) + .collect() +} + fn set_json_pointer(value: &mut Value, pointer: &str, replacement: Value) -> Result<()> { let target = value.pointer_mut(pointer).ok_or_else(|| eyre::eyre!("missing JSON pointer {pointer}"))?; @@ -461,6 +557,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { let graphiti_zep = find_by_field(adapters, "/adapter_id", "graphiti_zep_research_gate")?; let graphify = find_by_field(adapters, "/adapter_id", "graphify_docker_smoke")?; let qmd_deep = find_by_field(adapters, "/adapter_id", "qmd_deep_profile_gate")?; + let openviking_deep = find_by_field(adapters, "/adapter_id", "openviking_deep_profile_gate")?; assert_eq!(elf.pointer("/evidence_class").and_then(Value::as_str), Some("fixture_backed")); assert_eq!(elf.pointer("/overall_status").and_then(Value::as_str), Some("blocked")); @@ -470,17 +567,20 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { ); assert_eq!(elf_live.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); - assert_live_sweep_record(elf_live)?; + assert_live_sweep_record(elf_live, "blocked")?; assert_eq!(qmd.pointer("/overall_status").and_then(Value::as_str), Some("pass")); assert_eq!(qmd.pointer("/suites/0/status").and_then(Value::as_str), Some("not_encoded")); + + assert_qmd_live_baseline_record(qmd); + assert_eq!( qmd_live.pointer("/evidence_class").and_then(Value::as_str), Some("live_real_world") ); assert_eq!(qmd_live.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); - assert_live_sweep_record(qmd_live)?; + assert_live_sweep_record(qmd_live, "blocked")?; assert_eq!( agentmemory.pointer("/capabilities/1/status").and_then(Value::as_str), @@ -538,10 +638,51 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { qmd_deep.pointer("/capabilities/2/status").and_then(Value::as_str), Some("unsupported") ); + assert_eq!( + qmd_deep.pointer("/result/artifact").and_then(Value::as_str), + Some("docs/research/2026-06-11-qmd-openviking-strength-profile-report.json") + ); + assert_eq!( + openviking_deep.pointer("/adapter_kind").and_then(Value::as_str), + Some("docker_local_embed_context_trajectory_gate") + ); + + assert_openviking_deep_profile_gate(openviking_deep); + + assert_eq!( + openviking_deep.pointer("/result/artifact").and_then(Value::as_str), + Some("docs/research/2026-06-11-qmd-openviking-strength-profile-report.json") + ); Ok(()) } +fn assert_qmd_live_baseline_record(adapter: &Value) { + let result_evidence = adapter.pointer("/result/evidence").and_then(Value::as_str); + let retrieval_evidence = adapter.pointer("/suites/0/evidence").and_then(Value::as_str); + + assert!(result_evidence.is_some_and(|evidence| { + evidence.contains("This live_baseline_only record is same-corpus evidence only") + && evidence.contains("cite qmd_live_real_world for the full live real-world sweep") + && !evidence.contains("no real_world_job qmd adapter is encoded yet") + })); + assert!(retrieval_evidence.is_some_and(|evidence| { + evidence.contains("does not execute real_world_job retrieval prompts") + && evidence.contains("cite qmd_live_real_world for the live retrieval adapter run") + && !evidence.contains("no real_world_job retrieval adapter run is encoded") + })); +} + +fn assert_openviking_deep_profile_gate(adapter: &Value) { + let trajectory_evidence = adapter.pointer("/capabilities/1/evidence").and_then(Value::as_str); + + assert!(trajectory_evidence.is_some_and(|evidence| { + evidence.contains("evidence-bearing same-corpus output") + && evidence.contains("wrong_result missed-term evidence") + && !evidence.contains("setup reaches runnable OpenViking APIs") + })); +} + fn assert_first_generation_adapter_records( agentmemory: &Value, mem0: &Value, @@ -821,7 +962,7 @@ fn live_adapter_aggregate_forwards_graph_rag_smoke_controls() -> Result<()> { Ok(()) } -fn assert_live_sweep_record(adapter: &Value) -> Result<()> { +fn assert_live_sweep_record(adapter: &Value, production_ops_status: &str) -> Result<()> { let suites = array_at(adapter, "/suites")?; let capabilities = array_at(adapter, "/capabilities")?; let targeted = find_by_field(capabilities, "/capability", "targeted_live_pass")?; @@ -830,13 +971,37 @@ fn assert_live_sweep_record(adapter: &Value) -> Result<()> { let memory_evolution = find_by_field(suites, "/suite_id", "memory_evolution")?; let production_ops = find_by_field(suites, "/suite_id", "production_ops")?; let consolidation = find_by_field(suites, "/suite_id", "consolidation")?; + let knowledge = find_by_field(suites, "/suite_id", "knowledge_compilation")?; + let operator_debug = find_by_field(suites, "/suite_id", "operator_debugging_ux")?; + let capture = find_by_field(suites, "/suite_id", "capture_integration")?; + let personalization = find_by_field(suites, "/suite_id", "personalization")?; + let trust_sot = find_by_field(suites, "/suite_id", "trust_source_of_truth")?; + let retrieval = find_by_field(suites, "/suite_id", "retrieval")?; + let project_decisions = find_by_field(suites, "/suite_id", "project_decisions")?; + assert_eq!(suites.len(), 11); assert_eq!(targeted.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(full_pass.pointer("/status").and_then(Value::as_str), Some("wrong_result")); + assert!( + adapter + .pointer("/result/evidence") + .and_then(Value::as_str) + .is_some_and(|evidence| evidence.contains("38 jobs across all 11 encoded suites")) + ); + assert_eq!(trust_sot.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(work_resume.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(retrieval.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(project_decisions.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(memory_evolution.pointer("/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!(production_ops.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + production_ops.pointer("/status").and_then(Value::as_str), + Some(production_ops_status) + ); assert_eq!(consolidation.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(knowledge.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(operator_debug.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(capture.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(personalization.pointer("/status").and_then(Value::as_str), Some("pass")); Ok(()) } @@ -1124,6 +1289,572 @@ fn project_decisions_fixtures_report_decision_policy_cases() -> Result<()> { Ok(()) } +#[test] +fn qmd_openviking_strength_profile_report_preserves_claim_boundaries() -> Result<()> { + let report = + serde_json::from_str::(&fs::read_to_string(strength_profile_report_path()?)?)?; + let markdown = fs::read_to_string(strength_profile_markdown_path()?)?; + let readme = fs::read_to_string(readme_path()?)?; + let benchmarking_index = fs::read_to_string(benchmarking_index_path()?)?; + let iteration_direction = fs::read_to_string(iteration_direction_report_path()?)?; + + assert_strength_profile_summary(&report); + assert_strength_profile_terms(&report)?; + assert_qmd_strength_profile(&report)?; + assert_qmd_wrong_result_diagnosis(&report)?; + assert_openviking_strength_profile(&report)?; + assert_strength_profile_json_claim_boundaries(&report)?; + assert_strength_profile_markdown_boundaries(&markdown); + assert_operator_facing_strength_profile_boundaries( + &readme, + &benchmarking_index, + &iteration_direction, + ); + + Ok(()) +} + +#[test] +fn current_benchmark_reports_preserve_live_sweep_boundaries() -> Result<()> { + let measurement_audit = fs::read_to_string(measurement_coverage_audit_path()?)?; + let measurement_audit_json = serde_json::from_str::(&fs::read_to_string( + measurement_coverage_audit_json_path()?, + )?)?; + let competitor_matrix = fs::read_to_string(competitor_strength_matrix_path()?)?; + let competitor_matrix_json = serde_json::from_str::(&fs::read_to_string( + competitor_strength_matrix_json_path()?, + )?)?; + let external_manifest = fs::read_to_string(external_adapter_manifest_path())?; + let retrieval_debug_profile = + serde_json::from_str::(&fs::read_to_string(retrieval_debug_profile_json_path()?)?)?; + + assert!( + measurement_audit.contains( + "| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | `wrong_result:6` |" + ) + ); + assert!( + measurement_audit + .contains("qmd live fails 6/6 jobs after missing the delete/TTL tombstone evidence") + ); + assert!( + competitor_matrix + .contains("broader live suites remain `wrong_result`, `blocked`, or `not_encoded`") + ); + assert!(external_manifest.contains( + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible." + )); + assert!(external_manifest.contains( + "The qmd live real-world sweep covers the current encoded fixture corpus; expanded retrieval-debug strength suites still need their own materialized adapter run." + )); + + for stale_phrase in [ + "same live sweep shape as ELF", + "ELF and qmd live fail 5/6 jobs", + "both systems currently fail 5/6 live memory-evolution jobs", + "wrong_result, incomplete, blocked, and not_encoded states remain visible", + "broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`", + "The qmd live real-world slice covers representative jobs only", + ] { + assert!(!measurement_audit.contains(stale_phrase)); + assert!(!competitor_matrix.contains(stale_phrase)); + assert!(!external_manifest.contains(stale_phrase)); + } + + let qmd_live = find_by_field( + array_at(&measurement_audit_json, "/live_real_world_adapters")?, + "/adapter", + "qmd live CLI adapter", + )?; + + assert_eq!(qmd_live.pointer("/pass").and_then(Value::as_u64), Some(17)); + assert_eq!(qmd_live.pointer("/wrong_result").and_then(Value::as_u64), Some(6)); + assert_eq!(qmd_live.pointer("/expected_evidence_matched").and_then(Value::as_u64), Some(38)); + assert_eq!(qmd_live.pointer("/evidence_covered_count").and_then(Value::as_u64), Some(45)); + + let memory_evolution = find_by_field( + array_at(&measurement_audit_json, "/live_suite_breakdown")?, + "/suite", + "memory_evolution", + )?; + + assert_eq!( + memory_evolution.pointer("/elf_status_counts/wrong_result").and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + memory_evolution.pointer("/qmd_status_counts/wrong_result").and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + retrieval_debug_profile + .pointer("/live_real_world_full_sweep_context/qmd/pass") + .and_then(Value::as_u64), + Some(17) + ); + assert_eq!( + retrieval_debug_profile + .pointer("/live_real_world_full_sweep_context/qmd/wrong_result") + .and_then(Value::as_u64), + Some(6) + ); + + assert_competitor_strength_matrix_json(&competitor_matrix_json)?; + + Ok(()) +} + +fn assert_competitor_strength_matrix_json(matrix: &Value) -> Result<()> { + let projects = array_at(matrix, "/project_matrix")?; + let qmd = find_by_field(projects, "/project", "qmd")?; + let mem0 = find_by_field(projects, "/project", "mem0/OpenMemory")?; + let openviking = find_by_field(projects, "/project", "OpenViking")?; + + assert_eq!( + qmd.pointer("/current_evidence_class").and_then(Value::as_str), + Some("live_real_world") + ); + assert_eq!(qmd.pointer("/measured_status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + qmd.pointer("/unsupported_or_blocked_status/state").and_then(Value::as_str), + Some("not_encoded") + ); + assert!(qmd.pointer("/benchmark_before_claim").and_then(Value::as_str).is_some_and(|claim| { + claim.contains("before claiming ELF wins, ties, or loses on retrieval debugging") + })); + assert!( + qmd.pointer("/borrow_if_stronger") + .and_then(Value::as_str) + .is_some_and(|claim| claim.contains("transparent local knobs")) + ); + assert_eq!(mem0.pointer("/measured_status").and_then(Value::as_str), Some("pass")); + assert_eq!( + mem0.pointer("/unsupported_or_blocked_status/state").and_then(Value::as_str), + Some("not_encoded") + ); + assert!( + mem0.pointer("/benchmark_before_claim") + .and_then(Value::as_str) + .is_some_and(|claim| claim.contains("preference/entity history")) + ); + assert_eq!( + openviking.pointer("/current_evidence_class").and_then(Value::as_str), + Some("live_baseline_only") + ); + assert_eq!( + openviking.pointer("/measured_status").and_then(Value::as_str), + Some("wrong_result") + ); + assert_eq!( + openviking.pointer("/unsupported_or_blocked_status/state").and_then(Value::as_str), + Some("not_encoded") + ); + assert!( + openviking + .pointer("/unsupported_or_blocked_status/details") + .and_then(Value::as_str) + .is_some_and(|details| details.contains("same-corpus output misses expected evidence")) + ); + assert!( + openviking + .pointer("/benchmark_before_claim") + .and_then(Value::as_str) + .is_some_and(|claim| claim.contains("evidence-bearing same-corpus output pass")) + ); + + let scenarios = array_at(matrix, "/scenario_matrix")?; + let retrieval_debug = find_by_field(scenarios, "/scenario_id", "retrieval_debug")?; + let context_trajectory = find_by_field(scenarios, "/scenario_id", "context_trajectory")?; + + assert!( + retrieval_debug + .pointer("/current_state") + .and_then(Value::as_str) + .is_some_and(|state| state.contains("Measured tie on encoded retrieval answers")) + ); + assert!(retrieval_debug.pointer("/current_state").and_then(Value::as_str).is_some_and( + |state| state.contains("qmd remains stronger on local debug ergonomics not fully scored") + )); + assert!( + context_trajectory + .pointer("/current_state") + .and_then(Value::as_str) + .is_some_and(|state| state.contains("not a measured live winner")) + ); + assert!( + context_trajectory + .pointer("/next_measurement") + .and_then(Value::as_str) + .is_some_and(|measurement| measurement.contains("evidence-bearing retrieval pass")) + ); + + Ok(()) +} + +fn assert_strength_profile_summary(report: &Value) { + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.competitor_strength_profile_report/v1") + ); + assert_eq!( + report.pointer("/summary/qmd/retrieval_quality").and_then(Value::as_str), + Some("tie") + ); + assert_eq!( + report.pointer("/summary/qmd/local_query_transparency").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!( + report.pointer("/summary/qmd/local_replayability").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!( + report.pointer("/summary/qmd/overall_outcome").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!( + report.pointer("/summary/openviking/overall_outcome").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!( + report + .pointer("/qmd_strength_profile/win_tie_loss_summary/elf_win") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report.pointer("/qmd_strength_profile/win_tie_loss_summary/tie").and_then(Value::as_u64), + Some(3) + ); + assert_eq!( + report + .pointer("/qmd_strength_profile/win_tie_loss_summary/elf_loss") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/qmd_strength_profile/win_tie_loss_summary/not_tested") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/not_tested") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/openviking_context_trajectory_profile/win_tie_loss_summary/elf_win") + .and_then(Value::as_u64), + Some(1) + ); +} + +fn assert_strength_profile_terms(report: &Value) -> Result<()> { + let result_terms = array_at(report, "/result_type_terms")?; + let coverage_terms = array_at(report, "/coverage_status_terms")?; + let outcome_terms = array_at(report, "/outcome_terms")?; + let actual_result_terms = string_array_at(report, "/result_type_terms")?; + let actual_coverage_terms = string_array_at(report, "/coverage_status_terms")?; + + assert_eq!( + actual_result_terms, + [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported_claim", + ] + .map(str::to_owned) + ); + assert_eq!( + actual_coverage_terms, + [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported", + "unsupported_claim", + ] + .map(str::to_owned) + ); + assert!(!result_terms.iter().any(|term| term.as_str() == Some("unsupported"))); + assert!(!result_terms.iter().any(|term| term.as_str() == Some("partial"))); + assert!(!coverage_terms.iter().any(|term| term.as_str() == Some("partial"))); + assert!(result_terms.iter().any(|term| term.as_str() == Some("unsupported_claim"))); + assert!(coverage_terms.iter().any(|term| term.as_str() == Some("unsupported"))); + + assert_value_in_terms(report, "/summary/qmd/overall_outcome", outcome_terms)?; + assert_value_in_terms(report, "/summary/openviking/overall_outcome", outcome_terms)?; + + for scenario in array_at(report, "/qmd_strength_profile/scenario_outcomes")? { + assert_value_in_terms(scenario, "/result_type", result_terms)?; + assert_value_in_terms(scenario, "/elf_status", coverage_terms)?; + assert_value_in_terms(scenario, "/qmd_status", coverage_terms)?; + } + for scenario in array_at(report, "/openviking_context_trajectory_profile/scenario_outcomes")? { + assert_value_in_terms(scenario, "/result_type", result_terms)?; + assert_value_in_terms(scenario, "/openviking_status", coverage_terms)?; + assert_value_in_terms(scenario, "/elf_equivalent_status", coverage_terms)?; + } + + Ok(()) +} + +fn assert_value_in_terms(value: &Value, pointer: &str, terms: &[Value]) -> Result<()> { + let actual = value + .pointer(pointer) + .and_then(Value::as_str) + .ok_or_else(|| eyre::eyre!("missing string at {pointer}"))?; + + assert!( + terms.iter().any(|term| term.as_str() == Some(actual)), + "{actual} at {pointer} is not declared in the report term list" + ); + + Ok(()) +} + +fn assert_qmd_strength_profile(report: &Value) -> Result<()> { + let qmd_scenarios = array_at(report, "/qmd_strength_profile/scenario_outcomes")?; + let local_transparency = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-local-query-transparency")?; + let retrieval = find_by_field(qmd_scenarios, "/scenario_id", "qmd-retrieval-quality")?; + let rerank_controls = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-expansion-fusion-rerank-controls")?; + let stale_isolation = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-stale-context-isolation")?; + let lifecycle = find_by_field(qmd_scenarios, "/scenario_id", "qmd-update-delete-cold-start")?; + let operator_debug = + find_by_field(qmd_scenarios, "/scenario_id", "qmd-operator-debug-evidence")?; + let replayability = find_by_field(qmd_scenarios, "/scenario_id", "qmd-local-replayability")?; + let wrong_result = find_by_field(qmd_scenarios, "/scenario_id", "qmd-wrong-result-diagnosis")?; + + assert_eq!(qmd_scenarios.len(), 8); + assert_eq!(retrieval.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); + assert_eq!( + local_transparency.pointer("/elf_outcome").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!( + local_transparency.pointer("/result_type").and_then(Value::as_str), + Some("not_encoded") + ); + assert_eq!( + rerank_controls.pointer("/result_type").and_then(Value::as_str), + Some("not_encoded") + ); + assert_eq!(stale_isolation.pointer("/result_type").and_then(Value::as_str), Some("pass")); + assert_eq!(stale_isolation.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); + assert_eq!(lifecycle.pointer("/result_type").and_then(Value::as_str), Some("pass")); + assert_eq!(lifecycle.pointer("/elf_outcome").and_then(Value::as_str), Some("tie")); + assert_eq!(operator_debug.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(operator_debug.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); + assert_eq!(replayability.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(replayability.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); + assert_eq!( + wrong_result.pointer("/evidence_class").and_then(Value::as_str), + Some("research_gate") + ); + assert_eq!(wrong_result.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + + Ok(()) +} + +fn assert_qmd_wrong_result_diagnosis(report: &Value) -> Result<()> { + let taxonomy = array_at(report, "/qmd_strength_profile/wrong_result_diagnosis/taxonomy")?; + let absent = find_by_field(taxonomy, "/class", "evidence_absent")?; + let dropped = find_by_field(taxonomy, "/class", "retrieved_but_dropped")?; + let narrated = find_by_field(taxonomy, "/class", "selected_but_not_narrated")?; + let lifecycle = find_by_field(taxonomy, "/class", "contradicted_by_lifecycle_evidence")?; + + assert_eq!(absent.pointer("/coverage").and_then(Value::as_str), Some("observed")); + assert_eq!( + dropped.pointer("/coverage").and_then(Value::as_str), + Some("not_observed_candidate_trace_missing") + ); + assert_eq!(narrated.pointer("/coverage").and_then(Value::as_str), Some("observed")); + assert_eq!(lifecycle.pointer("/coverage").and_then(Value::as_str), Some("observed")); + + let qmd_diagnosis_jobs = array_at(report, "/qmd_strength_profile/wrong_result_diagnosis/jobs")?; + let delete_job = + find_by_field(qmd_diagnosis_jobs, "/job_id", "memory-evolution-delete-ttl-001")?; + + assert_eq!(qmd_diagnosis_jobs.len(), 6); + assert_eq!(delete_job.pointer("/qmd_status").and_then(Value::as_str), Some("wrong_result")); + assert!(array_contains_str(delete_job, "/missing_evidence", "delete-tombstone")?); + assert!( + delete_job + .pointer("/diagnosis") + .and_then(Value::as_str) + .is_some_and(|diagnosis| diagnosis.contains("typed wrong_result")) + ); + + Ok(()) +} + +fn assert_openviking_strength_profile(report: &Value) -> Result<()> { + let openviking_scenarios = + array_at(report, "/openviking_context_trajectory_profile/scenario_outcomes")?; + let trajectory = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-staged-retrieval-trajectory", + )?; + let precondition = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-evidence-bearing-retrieval-precondition", + )?; + let local_embed_setup = + find_by_field(openviking_scenarios, "/scenario_id", "openviking-local-embed-setup")?; + let missed_terms = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-missed-expected-terms-evidence", + )?; + let hierarchy = + find_by_field(openviking_scenarios, "/scenario_id", "openviking-hierarchy-selection")?; + let recursive_expansion = find_by_field( + openviking_scenarios, + "/scenario_id", + "openviking-recursive-context-expansion", + )?; + + assert_eq!(openviking_scenarios.len(), 6); + assert_eq!( + trajectory.pointer("/evidence_class").and_then(Value::as_str), + Some("research_gate") + ); + assert_eq!(trajectory.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(local_embed_setup.pointer("/result_type").and_then(Value::as_str), Some("pass")); + assert_eq!( + local_embed_setup.pointer("/elf_outcome").and_then(Value::as_str), + Some("not_tested") + ); + assert_eq!(local_embed_setup.pointer("/typed_blocker"), Some(&Value::Null)); + assert_eq!(precondition.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); + assert_eq!(precondition.pointer("/elf_outcome").and_then(Value::as_str), Some("elf_win")); + assert_eq!( + precondition.pointer("/typed_blocker").and_then(Value::as_str), + Some("output_missed_expected_terms") + ); + assert_eq!(missed_terms.pointer("/result_type").and_then(Value::as_str), Some("wrong_result")); + assert_eq!(missed_terms.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); + assert_eq!(hierarchy.pointer("/result_type").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(hierarchy.pointer("/elf_outcome").and_then(Value::as_str), Some("not_tested")); + assert_eq!( + recursive_expansion.pointer("/result_type").and_then(Value::as_str), + Some("not_encoded") + ); + assert_eq!( + recursive_expansion.pointer("/elf_outcome").and_then(Value::as_str), + Some("not_tested") + ); + + Ok(()) +} + +fn assert_strength_profile_json_claim_boundaries(report: &Value) -> Result<()> { + assert!(array_contains_str( + report, + "/claim_boundaries", + "ELF does not broadly beat qmd; it ties encoded retrieval and lifecycle correctness, keeps qmd query transparency as not_tested for comparative scoring, and leaves replayability not_tested." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "qmd expansion, fusion, and rerank superiority remains not_tested because the current qmd paths use --no-rerank and do not score internals." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "ELF does not beat OpenViking on context trajectory; OpenViking trajectory strengths remain not_tested behind a wrong_result same-corpus output precondition." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "Research_gate records are follow-up gates, not pass evidence." + )?); + assert!(array_contains_str( + report, + "/claim_boundaries", + "Missing equivalent surfaces are encoded as unsupported or not_encoded rather than fake losses." + )?); + + Ok(()) +} + +fn assert_strength_profile_markdown_boundaries(markdown: &str) { + assert!( + markdown.contains( + "| Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` |" + ) + ); + assert!( + markdown.contains("ELF ties qmd on the current encoded retrieval-correctness surfaces") + ); + assert!(markdown.contains("qmd remains the local retrieval-debug UX reference")); + assert!(markdown.contains("not scored as comparative ELF wins or losses")); + assert!(markdown.contains("ELF currently wins only the equivalent OpenViking same-corpus")); + assert!(markdown.contains("Do not claim ELF broadly beats qmd")); + assert!(markdown.contains( + "Do not claim ELF beats OpenViking on staged retrieval, hierarchy, or recursive" + )); + assert!(markdown.contains( + "Do not turn `research_gate`, `not_encoded`, or `unsupported` surfaces into wins" + )); + assert!(markdown.contains("no pass evidence is claimed")); + assert!(markdown.contains("typed `wrong_result` state")); +} + +fn assert_operator_facing_strength_profile_boundaries( + readme: &str, + benchmarking_index: &str, + iteration_direction: &str, +) { + assert!(readme.contains("Full-suite live real-world adapter sweep after XY-899")); + assert!(readme.contains("fresh ELF sweep reports 18 pass")); + assert!(readme.contains("5 wrong_result, 2 blocked, and 13 not_encoded jobs")); + assert!(readme.contains("fresh qmd sweep reports")); + assert!(readme.contains("17 pass, 6 wrong_result, 2 blocked, and 13 not_encoded jobs")); + assert!(readme.contains("The difference is the")); + assert!(readme.contains("delete/TTL tombstone case")); + assert!(readme.contains("qmd remains the local retrieval-debug UX reference")); + assert!(readme.contains("no broad ELF-over-qmd claim is allowed")); + assert!(readme.contains("qmd and OpenViking Strength-Profile Report - June 11, 2026")); + assert!(benchmarking_index.contains("2026-06-11-qmd-openviking-strength-profile-report.md")); + assert!( + benchmarking_index.contains("separates qmd retrieval quality from debug/replay ergonomics") + ); + assert!(benchmarking_index.contains("preserves OpenViking context-trajectory")); + assert!( + benchmarking_index + .contains("surfaces as `not_tested` until staged/hierarchical evidence is encoded") + ); + assert!( + iteration_direction + .contains("ELF and qmd are tied on the encoded live retrieval, work-resume, and") + ); + assert!(iteration_direction.contains("ELF does not yet beat qmd's local retrieval-debug")); + assert!( + iteration_direction + .contains("ELF beats OpenViking on context trajectory. That scenario is not encoded.") + ); + assert!( + iteration_direction + .contains("Do not promote a reference project into a win/loss claim until") + ); +} + #[test] fn generated_json_report_renders_markdown() -> Result<()> { let report = run_json_report()?; diff --git a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md index c5f988bb..5a20aacf 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md +++ b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md @@ -72,10 +72,10 @@ sweeps for ELF and qmd: Interpretation: - This is a near tie for the currently encoded live real-world sweep, with ELF one - job ahead in this fresh run. + job ahead in this fresh run because qmd misses the delete/TTL tombstone job. - Both pass `trust_source_of_truth`, `work_resume`, `project_decisions`, `retrieval`, and `personalization`. -- Both fail `memory_evolution` live conflict evidence with `wrong_result`. +- Both fail most `memory_evolution` live conflict evidence with `wrong_result`. - Both leave consolidation, knowledge compilation, operator debugging, capture integration, and production-ops operator boundaries as `not_encoded` or `blocked`. @@ -245,8 +245,9 @@ These are needed for broad credibility but should not block personal production Do not claim: -- ELF beats qmd overall. Current live sweep is essentially tied, and qmd still owns - stronger local retrieval-debug ergonomics. +- ELF beats qmd overall. ELF is one pass ahead in the fresh aggregate because qmd + misses the delete/TTL tombstone job, but neither adapter has full-suite live pass + evidence and qmd still owns stronger local retrieval-debug ergonomics. - ELF has full-suite live real-world pass evidence. It does not. - ELF has private-corpus production quality proof. The private profile currently fails closed without an operator-owned manifest. diff --git a/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md b/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md index 6e3af93e..8054b3fe 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md +++ b/docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md @@ -67,11 +67,13 @@ Full live sweep context remains a non-pass for both systems: | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `5.823 ms` | -| qmd live CLI adapter | `38` | `18` | `5` | `2` | `13` | `0.512` | `705.877 ms` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `691.163 ms` | Do not overread the latency row. The ELF adapter is a service-runtime path and the qmd adapter is a CLI materialization path; the row is useful as observed harness evidence, -not as an apples-to-apples product latency benchmark. +not as an apples-to-apples product latency benchmark. The aggregate pass-count +difference comes from the memory-evolution delete/TTL tombstone job; it does not erase +qmd's local retrieval-debug ergonomics advantage. ## Stress Baseline diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index ea0636aa..584b3142 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -24,10 +24,9 @@ What is proven today: - ELF has a strong fixture-backed real-world benchmark contract: 38 jobs, 36 pass, 2 blocked operator boundaries, and no wrong results in the fixture aggregate. -- ELF and qmd have comparable full-suite live real-world sweeps. The latest generated - artifacts are close but no longer identical: ELF has 38 jobs with 18 pass, - 5 wrong_result, 2 blocked, and 13 not_encoded, while qmd has 17 pass, - 6 wrong_result, 2 blocked, and 13 not_encoded. +- ELF and qmd have comparable full-suite live real-world sweeps, but neither has a + full-suite live pass. ELF is one pass ahead in the fresh aggregate because qmd + misses the memory-evolution delete/TTL tombstone job. - ELF is ahead on production-operation evidence among tracked systems because it has checked-in provider synthetic, stress, backfill, backup/restore, and Qdrant rebuild evidence. @@ -85,13 +84,13 @@ live adapter or competitor runtime can complete those jobs. | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | Evidence recall | Evidence coverage | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `6.761 ms` | `41/77` | `48/84` | -| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `842.057 ms` | `38/77` | `45/84` | +| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `5.100 ms` | `41/77` | `48/84` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `691.163 ms` | `38/77` | `45/84` | This supports a near tie on the currently encoded live real-world suite shape, with -ELF one job ahead in this fresh run. It does not support a broad ELF-over-qmd claim -because qmd remains the stronger retrieval-debug UX reference and its deep profile is -still not encoded. +ELF one job ahead because qmd misses the delete/TTL tombstone case. It does not +support a broad ELF-over-qmd claim because qmd remains the stronger retrieval-debug UX +reference and its deep profile is still not encoded. ### Live Suite Breakdown @@ -113,9 +112,10 @@ ELF and qmd have the same status shape outside `memory_evolution`. The differenc | `operator_debugging_ux` | `1` | `not_encoded:1` | `not_encoded:1` | | `production_ops` | `6` | `blocked:2`, `not_encoded:4` | `blocked:2`, `not_encoded:4` | -The live wrong results are all memory-evolution jobs. The live adapters retrieve -current evidence but do not yet provide all required historical conflict evidence -links for current-vs-historical reasoning. +The ELF live wrong results are five memory-evolution jobs. qmd has those same conflict +evidence failures plus the delete/TTL tombstone miss. The live adapters retrieve +current evidence in several cases but do not yet provide the required historical +conflict evidence links for current-vs-historical reasoning. ## External Adapter Ledger @@ -137,15 +137,15 @@ The checked-in manifest records 21 adapter records across 17 unique project name | `not_encoded` | `7` | The generated JSON report emits `external_project_count: 16`, matching the unique -non-ELF project-name count from the manifest. The full project-name count remains 17 -when ELF is included. +non-ELF project-name count from the manifest. The companion audit JSON separately +records `unique_project_names: 17` for the full project list including ELF. ## Project Coverage | Project | Best current evidence | Current measured state | Strongest unproven scenario | Next measurement before claim | | --- | --- | --- | --- | --- | | ELF | `fixture_backed` plus `live_real_world` | Fixture aggregate passes except 2 blocked operator boundaries; live full sweep is `wrong_result`. | Full live memory evolution, live consolidation, live knowledge pages, live capture, live production ops. | Memory-evolution diagnostic report, then live operator/capture/consolidation reports. | -| qmd | `live_real_world` plus `live_baseline_only` | Same-corpus baseline passes; current live sweep is one memory-evolution job behind ELF. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | +| qmd | `live_real_world` plus `live_baseline_only` | Fresh full sweep is one pass behind ELF because qmd misses the delete/TTL tombstone job; same-corpus baseline passes. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | | agentmemory | `live_baseline_only` | `lifecycle_fail`. | Durable coding-agent continuity and capture hooks. | Durable lifecycle and work-resume/capture adapter report. | | mem0/OpenMemory | `live_baseline_only` | Basic local smoke now passes; history/UI/hosted/graph behavior remains `not_encoded`. | Entity history, lifecycle UI, OpenMemory inspection. | Entity-history, deletion-audit, and UI/export readback report. | | memsearch | `live_baseline_only` | Basic canonical Markdown reindex/reload smoke now passes; real-world prompt coverage remains `not_encoded`. | Markdown canonical store and local reindex clarity. | Source-of-truth and retrieval-debug real-world adapter report. | @@ -170,7 +170,7 @@ when ELF is included. | Work resume | ELF and qmd live pass. | ELF is credible on encoded work resume. | agentmemory, claude-mem, and OpenViking comparable continuity adapters. | | Project decisions | ELF and qmd live pass. | ELF is credible on encoded project-decision recovery. | Letta core/archival decision memory comparison. | | Source of truth | ELF and qmd live pass; ELF has stronger production restore/rebuild evidence. | ELF has strongest measured source-of-truth discipline. | memsearch source-of-truth reindex/reload evidence. | -| Memory evolution | ELF and qmd live fail 5/6 jobs; fixture aggregate passes. | No live superiority claim. | Historical conflict evidence links and Graphiti/Zep temporal comparison. | +| Memory evolution | ELF live fails 5/6 jobs; qmd live fails 6/6 jobs after missing the delete/TTL tombstone evidence; fixture aggregate passes. | No broad live superiority claim. | Historical conflict evidence links and Graphiti/Zep temporal comparison. | | Consolidation | Fixture aggregate passes; live adapters are not encoded. | Fixture-only claim. | Live proposal generation with lineage, confidence, and review-action audit. | | Knowledge pages | Fixture aggregate passes; live adapters are not encoded. | Fixture-only claim. | Live page rebuild/lint plus llm-wiki, gbrain, GraphRAG, and graphify comparisons. | | Operator debugging | Fixture aggregate passes; live adapters are not encoded. | Fixture-only claim. | Trace hydration, stage attribution, dropped-candidate, and repair-action scoring. | @@ -192,7 +192,8 @@ Order these by decision value, not implementation convenience: rerank, dropped candidates, and command-line replay. 2. ELF/qmd live memory-evolution diagnostic - - Why: both systems currently fail 5/6 live memory-evolution jobs. + - Why: ELF currently fails 5/6 live memory-evolution jobs and qmd fails 6/6, + including the delete/TTL tombstone case. - Output: per-job evidence-link failure analysis for current-vs-historical facts, supersession, and relation temporal validity. @@ -219,9 +220,9 @@ Order these by decision value, not implementation convenience: - Output: Docker-contained artifacts mapped to evidence ids, or typed setup and resource blockers. -Before publishing the next aggregate report, keep the generated `external_project_count` -field tied to unique non-ELF project names so readers do not confuse adapter records -with unique external projects. +Before publishing the next aggregate report, keep `external_project_count` aligned +with unique non-ELF project names so readers do not confuse project coverage with +adapter-record coverage. ## Fail Criteria diff --git a/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md new file mode 100644 index 00000000..99b1260a --- /dev/null +++ b/docs/guide/benchmarking/2026-06-11-qmd-openviking-strength-profile-report.md @@ -0,0 +1,127 @@ +# qmd and OpenViking Strength-Profile Report - June 11, 2026 + +Goal: Compare ELF against qmd and OpenViking on their actual strengths without +turning broad live-sweep or smoke results into unsupported win claims. +Read this when: You need the XY-899 scenario-level qmd retrieval-debug and +OpenViking context-trajectory benchmark/report outcome. +Inputs: The June 11 retrieval-debug, memory-evolution, and temporal-history reports, +the real-world benchmark spec, the external adapter manifest, and +`scripts/real-world-live-adapters.sh`. +Outputs: Scenario-level win/tie/loss/not-tested judgments, qmd wrong-result +diagnosis taxonomy, OpenViking typed trajectory blockers, and claim boundaries. + +Machine-readable companion: +`docs/research/2026-06-11-qmd-openviking-strength-profile-report.json`. + +## Executive Judgment + +ELF does not have a broad win against either qmd or OpenViking on their strengths. + +The measured qmd judgment is narrower: + +- Retrieval quality: `tie`. ELF and qmd both pass the encoded live real-world + retrieval suite and both pass the 480-document stress retrieval baseline. +- Local query transparency: `not_tested`. qmd's current artifacts expose directly + inspectable top-10 JSON rows with files, line numbers, snippets, and scores. ELF + has stronger service traces and production-operation evidence, but the checked-in + stress report does not hydrate an equivalent candidate list, so no scored ELF loss + is claimed for this surface. +- Local replayability: `not_tested`. qmd has a concise observed CLI replay path, and + ELF has service traces plus admin bundle endpoints, but no scored replayability rule + compares those surfaces yet. +- Expansion/fusion/rerank controls: `not_tested`. The current qmd materializer and + stress run use `--no-rerank`; no scored expansion/fusion/rerank profile exists. + +The measured OpenViking judgment is split by surface: + +- Same-corpus evidence-bearing preconditions: `elf_win`. The pinned Docker local + embedding path reaches `add_resource`/`find`, but the OpenViking smoke remains + `wrong_result` because expected evidence terms are missed while ELF passes the + equivalent retrieval precondition. +- Context trajectory strengths: `not_tested`. The current OpenViking wrong-result + smoke is not a scored staged-trajectory comparison. +- Staged retrieval, hierarchy selection, and recursive/context expansion remain + `research_gate` / `not_encoded`; no ELF win, tie, or loss is claimed against those + strengths. + +## qmd Scenario Outcomes + +| Scenario | Evidence Class | Result Type | ELF Outcome | What It Means | +| --- | --- | --- | --- | --- | +| Retrieval quality | `live_real_world` | `pass` | `tie` | Both systems pass 5/5 live retrieval jobs with 6/6 expected evidence matched. | +| Local query transparency | `live_baseline_only` | `not_encoded` | `not_tested` | qmd exposes top-10 files, line numbers, snippets, scores, and distractor density directly in the stress artifact, but the equivalent ELF candidate-list surface is not encoded. | +| Expansion/fusion/rerank controls | `research_gate` | `not_encoded` | `not_tested` | No scored profile proves either system's expansion, fusion, or rerank superiority. | +| Stale context isolation | `live_real_world` | `pass` | `tie` | Both systems pass the encoded current-vs-obsolete and distractor-heavy retrieval jobs. | +| Update/delete/cold-start behavior | `live_baseline_only` | `pass` | `tie` | Equivalent update replacement, delete suppression, and cold-start recovery checks pass for both. | +| Operator-debug evidence | `live_real_world` | `not_encoded` | `not_tested` | The live sweep marks operator-debugging UX `not_encoded` for both systems. | +| Local replayability | `live_baseline_only` | `not_encoded` | `not_tested` | qmd has a shorter observed CLI replay path, but no scored replayability rule compares it with ELF's trace/admin replay surfaces yet. | +| Wrong-result diagnosis | `research_gate` | `not_encoded` | `not_tested` | The report classifies qmd memory-evolution failures, but qmd candidate-drop traces are not yet materialized and no pass evidence is claimed. | + +Summary: qmd strength-profile outcomes are `0` ELF wins, `3` ties, `0` ELF losses, +and `5` not-tested scenarios. This distinguishes retrieval quality from +debug/replay ergonomics: the retrieval result is tied, qmd remains the local +retrieval-debug UX reference, and query transparency plus replayability remain +unscored for comparative ELF win/loss claims. + +## qmd Wrong-Result Diagnosis + +The report adds a qmd diagnosis taxonomy with four classes: + +| Diagnosis Class | Current qmd Coverage | +| --- | --- | +| `evidence_absent` | Observed on the verdict caveat, preference rationale, and delete tombstone misses. | +| `retrieved_but_dropped` | Defined but not observed because current qmd live job artifacts do not expose candidate-stage traces. | +| `selected_but_not_narrated` | Observed on supersession jobs where qmd had evidence but did not narrate current-vs-historical state. | +| `contradicted_by_lifecycle_evidence` | Observed when current, historical, supersession, or tombstone evidence keeps the answer in typed `wrong_result` state. | + +The key qmd memory-evolution diagnosis is unchanged from the June 11 diagnostic: +qmd is `0/6` pass on live memory-evolution, misses three required evidence links, +and fails the delete/TTL tombstone job. The new report records that as typed +diagnosis evidence, not as a broad ELF-over-qmd claim. + +## OpenViking Scenario Outcomes + +| Scenario | Evidence Class | Result Type | ELF Outcome | Typed Blocker | +| --- | --- | --- | --- | --- | +| Docker local embedding setup | `live_baseline_only` | `pass` | `not_tested` | none | +| Same-corpus evidence-bearing retrieval precondition | `live_baseline_only` | `wrong_result` | `elf_win` | `output_missed_expected_terms` | +| Staged retrieval trajectory | `research_gate` | `not_encoded` | `not_tested` | `needs_evidence_bearing_same_corpus_output` | +| Hierarchy selection | `research_gate` | `not_encoded` | `not_tested` | `hierarchy_output_not_scored` | +| Recursive/context expansion | `research_gate` | `not_encoded` | `not_tested` | `recursive_expansion_not_materialized` | +| Missed expected terms evidence | `live_baseline_only` | `wrong_result` | `not_tested` | `retrieval_wrong_result` | + +Summary: OpenViking profile outcomes are `1` ELF win, `0` ties, `0` ELF losses, and +`5` not-tested scenarios. The single win is only the same-corpus evidence-bearing +precondition. The current smoke wrong-result is useful typed failure evidence, but it +is not a second comparative win and not a scored staged-trajectory comparison, so +context-trajectory strengths remain not tested. + +## Claim Boundaries + +Allowed: + +- ELF ties qmd on the current encoded retrieval-correctness surfaces. +- qmd remains the local retrieval-debug UX reference on the currently evidenced query + transparency artifact ergonomics; query transparency and replayability are observed + but not scored as comparative ELF wins or losses. +- qmd expansion/fusion/rerank superiority is untested. +- OpenViking's Docker local embedding setup reaches runtime, but context trajectory + remains untested because evidence-bearing same-corpus retrieval is not passing. +- ELF currently wins only the equivalent OpenViking same-corpus retrieval + precondition surface, not OpenViking's staged trajectory strengths. + +Not allowed: + +- Do not claim ELF broadly beats qmd. +- Do not claim qmd's debug ergonomics are equivalent to retrieval quality. +- Do not claim ELF beats OpenViking on staged retrieval, hierarchy, or recursive + context expansion. +- Do not turn `research_gate`, `not_encoded`, or `unsupported` surfaces into wins or + losses. + +## Validation Hook + +The checked-in consistency test reads the machine-readable companion report and +asserts the qmd/OpenViking scenario counts, diagnosis taxonomy, and bottom-line +claim boundaries. This keeps future report edits from silently converting untested +strength surfaces into pass claims. diff --git a/docs/guide/benchmarking/index.md b/docs/guide/benchmarking/index.md index c4b979d2..b6ab2b53 100644 --- a/docs/guide/benchmarking/index.md +++ b/docs/guide/benchmarking/index.md @@ -70,6 +70,10 @@ cleanup, use `docs/guide/single_user_production.md`. records Graphiti/Zep and Letta claim boundaries, and turns qmd, mem0/OpenMemory, Graphiti/Zep, Letta, and adjacent project strengths into benchmark-gated ELF optimization directions. +- `2026-06-11-qmd-openviking-strength-profile-report.md`: XY-899 strength-profile + report that separates qmd retrieval quality from debug/replay ergonomics, records + qmd wrong-result diagnosis classes, and preserves OpenViking context-trajectory + surfaces as `not_tested` until staged/hierarchical evidence is encoded. - `2026-06-11-first-generation-oss-adapter-promotion-report.md`: XY-898 first-generation OSS adapter promotion report that updates agentmemory, mem0/OpenMemory, memsearch, and claude-mem with fresh scenario-level baseline diff --git a/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json b/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json index fed5fed9..72f22936 100644 --- a/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json +++ b/docs/research/2026-06-11-elf-qmd-retrieval-debug-profile.json @@ -48,12 +48,12 @@ }, "qmd": { "job_count": 38, - "pass": 18, - "wrong_result": 5, + "pass": 17, + "wrong_result": 6, "blocked": 2, "not_encoded": 13, - "mean_score": 0.512, - "mean_latency_ms": 705.877 + "mean_score": 0.486, + "mean_latency_ms": 691.163 } }, "stress_baseline": { diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index 3f6c7965..d11270f4 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -60,84 +60,82 @@ "blocked": 2, "not_encoded": 13, "mean_score": 0.486, - "mean_latency_ms": 842.057, + "mean_latency_ms": 691.163, "expected_evidence_total": 77, "expected_evidence_matched": 38, "evidence_required_count": 84, "evidence_covered_count": 45 } ], - "live_suite_breakdown": { - "delta": "ELF passes memory-evolution-delete-ttl-001 while qmd reports wrong_result; other suite status shapes match.", - "suites": [ - { - "suite": "trust_source_of_truth", - "jobs": 1, - "elf_status_counts": {"pass": 1}, - "qmd_status_counts": {"pass": 1} - }, - { - "suite": "work_resume", - "jobs": 5, - "elf_status_counts": {"pass": 5}, - "qmd_status_counts": {"pass": 5} - }, - { - "suite": "retrieval", - "jobs": 5, - "elf_status_counts": {"pass": 5}, - "qmd_status_counts": {"pass": 5} - }, - { - "suite": "project_decisions", - "jobs": 5, - "elf_status_counts": {"pass": 5}, - "qmd_status_counts": {"pass": 5} - }, - { - "suite": "personalization", - "jobs": 1, - "elf_status_counts": {"pass": 1}, - "qmd_status_counts": {"pass": 1} - }, - { - "suite": "memory_evolution", - "jobs": 6, - "elf_status_counts": {"pass": 1, "wrong_result": 5}, - "qmd_status_counts": {"wrong_result": 6} - }, - { - "suite": "capture_integration", - "jobs": 2, - "elf_status_counts": {"not_encoded": 2}, - "qmd_status_counts": {"not_encoded": 2} - }, - { - "suite": "consolidation", - "jobs": 4, - "elf_status_counts": {"not_encoded": 4}, - "qmd_status_counts": {"not_encoded": 4} - }, - { - "suite": "knowledge_compilation", - "jobs": 2, - "elf_status_counts": {"not_encoded": 2}, - "qmd_status_counts": {"not_encoded": 2} - }, - { - "suite": "operator_debugging_ux", - "jobs": 1, - "elf_status_counts": {"not_encoded": 1}, - "qmd_status_counts": {"not_encoded": 1} - }, - { - "suite": "production_ops", - "jobs": 6, - "elf_status_counts": {"blocked": 2, "not_encoded": 4}, - "qmd_status_counts": {"blocked": 2, "not_encoded": 4} - } - ] - }, + "live_suite_delta": "ELF passes memory-evolution-delete-ttl-001 while qmd reports wrong_result; other suite status shapes match.", + "live_suite_breakdown": [ + { + "suite": "trust_source_of_truth", + "jobs": 1, + "elf_status_counts": {"pass": 1}, + "qmd_status_counts": {"pass": 1} + }, + { + "suite": "work_resume", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "retrieval", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "project_decisions", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "personalization", + "jobs": 1, + "elf_status_counts": {"pass": 1}, + "qmd_status_counts": {"pass": 1} + }, + { + "suite": "memory_evolution", + "jobs": 6, + "elf_status_counts": {"pass": 1, "wrong_result": 5}, + "qmd_status_counts": {"wrong_result": 6} + }, + { + "suite": "capture_integration", + "jobs": 2, + "elf_status_counts": {"not_encoded": 2}, + "qmd_status_counts": {"not_encoded": 2} + }, + { + "suite": "consolidation", + "jobs": 4, + "elf_status_counts": {"not_encoded": 4}, + "qmd_status_counts": {"not_encoded": 4} + }, + { + "suite": "knowledge_compilation", + "jobs": 2, + "elf_status_counts": {"not_encoded": 2}, + "qmd_status_counts": {"not_encoded": 2} + }, + { + "suite": "operator_debugging_ux", + "jobs": 1, + "elf_status_counts": {"not_encoded": 1}, + "qmd_status_counts": {"not_encoded": 1} + }, + { + "suite": "production_ops", + "jobs": 6, + "elf_status_counts": {"blocked": 2, "not_encoded": 4}, + "qmd_status_counts": {"blocked": 2, "not_encoded": 4} + } + ], "adapter_ledger": { "adapter_records": 21, "unique_project_names": 17, @@ -158,7 +156,7 @@ "xy900_update_note": "XY-900 promotes graphify from research_gate/blocked to a tiny scored live_real_world wrong_result smoke; broad graph/RAG quality remains unproven." }, "claim_boundary": { - "elf_vs_qmd": "near_tie_on_current_encoded_live_real_world_shape_not_overall_win", + "elf_vs_qmd": "near_tie_with_narrow_delete_ttl_elf_lead_not_overall_win", "elf_personal_production": "credible_with_bounded_caveats", "broad_competitor_superiority": "not_proven", "major_unmeasured_strengths": [ diff --git a/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json new file mode 100644 index 00000000..d8d966d6 --- /dev/null +++ b/docs/research/2026-06-11-qmd-openviking-strength-profile-report.json @@ -0,0 +1,376 @@ +{ + "schema": "elf.competitor_strength_profile_report/v1", + "run_id": "2026-06-11-qmd-openviking-strength-profile", + "created_at": "2026-06-11", + "scope": "Scenario-level qmd retrieval-debug and OpenViking context-trajectory strength profile outcomes for XY-899.", + "inputs": [ + "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md", + "docs/guide/benchmarking/2026-06-11-elf-qmd-memory-evolution-diagnostic.md", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md", + "docs/spec/real_world_agent_memory_benchmark_v1.md", + "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json", + "scripts/real-world-live-adapters.sh" + ], + "outcome_terms": [ + "elf_win", + "tie", + "elf_loss", + "not_tested" + ], + "result_type_terms": [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported_claim" + ], + "coverage_status_terms": [ + "pass", + "wrong_result", + "blocked", + "incomplete", + "lifecycle_fail", + "not_encoded", + "unsupported", + "unsupported_claim" + ], + "evidence_class_terms": [ + "fixture_backed", + "live_baseline_only", + "live_real_world", + "research_gate" + ], + "summary": { + "qmd": { + "overall_outcome": "not_tested", + "overall_rationale": "ELF ties qmd on encoded retrieval and lifecycle surfaces; qmd query-transparency, replayability, and expansion/fusion/rerank strengths remain not_tested for comparative scoring because equivalent scored ELF surfaces are not encoded.", + "retrieval_quality": "tie", + "local_query_transparency": "not_tested", + "local_replayability": "not_tested", + "expansion_fusion_rerank": "not_tested", + "claim": "ELF ties qmd on encoded retrieval correctness and equivalent update/delete/cold-start behavior. qmd remains the local retrieval-debug UX reference, but ELF has no scored loss on query-transparency, replayability, expansion, fusion, or rerank controls until equivalent comparative surfaces are encoded." + }, + "openviking": { + "overall_outcome": "not_tested", + "overall_rationale": "OpenViking context-trajectory strengths remain not_tested; ELF has only one same-corpus retrieval precondition win.", + "claim": "ELF has one measured win on the same-corpus evidence-bearing precondition where OpenViking currently returns wrong_result. ELF does not have a measured win, tie, or loss against OpenViking context-trajectory strengths because staged trajectory, hierarchy selection, and recursive expansion remain research-gate/not_encoded." + } + }, + "qmd_strength_profile": { + "scenario_outcomes": [ + { + "scenario_id": "qmd-retrieval-quality", + "surface": "retrieval quality", + "evidence_class": "live_real_world", + "result_type": "pass", + "elf_status": "pass", + "qmd_status": "pass", + "elf_outcome": "tie", + "retrieval_quality": "ELF and qmd each pass 5/5 live real-world retrieval jobs with 6/6 expected evidence matched.", + "debug_replay_ergonomics": "not scored by this scenario", + "source_artifacts": [ + "tmp/real-world-memory/live-adapters/elf-report.json", + "tmp/real-world-memory/live-adapters/qmd-report.json", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-local-query-transparency", + "surface": "local query transparency", + "evidence_class": "live_baseline_only", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "pass", + "elf_outcome": "not_tested", + "retrieval_quality": "not a correctness scenario", + "debug_replay_ergonomics": "qmd stress artifacts expose per-query top-10 files, line numbers, snippets, scores, and distractor density; ELF stress artifacts expose trace ids and top evidence but do not hydrate an equivalent candidate list in the checked-in report, so this surface is not scored as a comparative ELF loss.", + "source_artifacts": [ + "scripts/live-baseline-benchmark.sh", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-expansion-fusion-rerank-controls", + "surface": "expansion, fusion, and rerank controls", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "not_encoded", + "elf_outcome": "not_tested", + "retrieval_quality": "not scored", + "debug_replay_ergonomics": "The qmd materializer and stress baseline use structured lex/vec query input with --no-rerank; no scenario scores expansion, fusion, or rerank superiority for either system.", + "source_artifacts": [ + "scripts/real-world-live-adapters.sh", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-stale-context-isolation", + "surface": "stale context isolation", + "evidence_class": "live_real_world", + "result_type": "pass", + "elf_status": "pass", + "qmd_status": "pass", + "elf_outcome": "tie", + "retrieval_quality": "Both adapters pass the encoded retrieval current-vs-obsolete and distractor-heavy jobs.", + "debug_replay_ergonomics": "The debug explanation of stale-candidate rejection is not scored beyond the job answer and evidence match.", + "source_artifacts": [ + "apps/elf-eval/fixtures/real_world_memory/retrieval/current_vs_obsolete.json", + "apps/elf-eval/fixtures/real_world_memory/retrieval/distractor_heavy.json", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-update-delete-cold-start", + "surface": "update, delete, and cold-start behavior", + "evidence_class": "live_baseline_only", + "result_type": "pass", + "elf_status": "pass", + "qmd_status": "pass", + "elf_outcome": "tie", + "retrieval_quality": "Equivalent qmd and ELF stress-baseline lifecycle checks pass for update replacement, delete suppression, and cold-start recovery.", + "debug_replay_ergonomics": "ELF has additional service lifecycle, backfill, rebuild, and resource evidence, but the equivalent qmd strength surface is a tie.", + "source_artifacts": [ + "tmp/live-baseline/live-baseline-report.json", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-operator-debug-evidence", + "surface": "operator-debug evidence", + "evidence_class": "live_real_world", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "not_encoded", + "elf_outcome": "not_tested", + "retrieval_quality": "not scored", + "debug_replay_ergonomics": "The live real-world sweep marks operator_debugging_ux not_encoded for both ELF and qmd. ELF fixture-backed operator-debug jobs pass, but they are not live adapter evidence.", + "source_artifacts": [ + "tmp/real-world-memory/live-adapters/elf-report.json", + "tmp/real-world-memory/live-adapters/qmd-report.json", + "apps/elf-eval/fixtures/real_world_memory/retrieval/stage_explainability_wrong_result.json" + ] + }, + { + "scenario_id": "qmd-local-replayability", + "surface": "local replayability", + "evidence_class": "live_baseline_only", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "pass", + "elf_outcome": "not_tested", + "retrieval_quality": "not a correctness scenario", + "debug_replay_ergonomics": "qmd's observed replay path is collection add, update, embed -f, and query --json in a fresh CLI process; ELF has service traces and admin bundle endpoints, but no scored replayability rule compares the two surfaces yet.", + "source_artifacts": [ + "scripts/live-baseline-benchmark.sh", + "docs/guide/benchmarking/2026-06-11-elf-qmd-retrieval-debug-profile.md" + ] + }, + { + "scenario_id": "qmd-wrong-result-diagnosis", + "surface": "wrong-result diagnosis", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "elf_status": "not_encoded", + "qmd_status": "not_encoded", + "elf_outcome": "not_tested", + "retrieval_quality": "The memory-evolution diagnostic classifies qmd misses and selected-but-not-narrated lifecycle failures from produced evidence; candidate-drop classification remains untested because qmd live job artifacts do not expose candidate-stage traces.", + "debug_replay_ergonomics": "The report taxonomy supports absent evidence, retrieved-but-dropped evidence, selected-but-not-narrated evidence, and lifecycle-contradicted evidence. Current qmd data exercises absent and selected-but-not-narrated classes; retrieved-but-dropped remains not observed.", + "source_artifacts": [ + "docs/guide/benchmarking/2026-06-11-elf-qmd-memory-evolution-diagnostic.md" + ] + } + ], + "win_tie_loss_summary": { + "elf_win": 0, + "tie": 3, + "elf_loss": 0, + "not_tested": 5 + }, + "wrong_result_diagnosis": { + "taxonomy": [ + { + "class": "evidence_absent", + "meaning": "Required evidence is absent from the adapter-produced evidence ids.", + "coverage": "observed" + }, + { + "class": "retrieved_but_dropped", + "meaning": "Required evidence appears in an intermediate candidate set but is absent from the final selected/narrated answer.", + "coverage": "not_observed_candidate_trace_missing" + }, + { + "class": "selected_but_not_narrated", + "meaning": "Evidence is selected or available, but the answer does not narrate the required current-vs-historical or lifecycle relationship.", + "coverage": "observed" + }, + { + "class": "contradicted_by_lifecycle_evidence", + "meaning": "The answer is contradicted or made incomplete by available current, historical, supersession, or tombstone evidence.", + "coverage": "observed" + } + ], + "jobs": [ + { + "job_id": "memory-evolution-benchmark-verdict-001", + "qmd_status": "wrong_result", + "score": 0.15, + "classifications": [ + "evidence_absent", + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [ + "verdict-bounded-private-caveat" + ], + "diagnosis": "qmd missed the caveat evidence and did not represent the superseded not-ready verdict as historical." + }, + { + "job_id": "memory-evolution-deploy-method-001", + "qmd_status": "wrong_result", + "score": 0.4, + "classifications": [ + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [], + "diagnosis": "qmd retrieved current runbook and rationale evidence, but did not preserve the old quickstart path as historical." + }, + { + "job_id": "memory-evolution-issue-state-001", + "qmd_status": "wrong_result", + "score": 0.4, + "classifications": [ + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [], + "diagnosis": "qmd found current done state and rationale evidence, but did not surface the earlier blocked state as superseded history." + }, + { + "job_id": "memory-evolution-preference-001", + "qmd_status": "wrong_result", + "score": 0.15, + "classifications": [ + "evidence_absent", + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [ + "pref-current-concise-rationale" + ], + "diagnosis": "qmd only returned rationale evidence and did not preserve the old terse preference as historical." + }, + { + "job_id": "memory-evolution-relation-temporal-001", + "qmd_status": "wrong_result", + "score": 0.35, + "classifications": [ + "selected_but_not_narrated", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [], + "diagnosis": "qmd retrieved current and historical owners, but did not produce temporal-validity explanation or update rationale." + }, + { + "job_id": "memory-evolution-delete-ttl-001", + "qmd_status": "wrong_result", + "score": 0.5, + "classifications": [ + "evidence_absent", + "contradicted_by_lifecycle_evidence" + ], + "missing_evidence": [ + "delete-tombstone" + ], + "diagnosis": "qmd retrieved the current plan but missed the tombstone evidence, so the delete/TTL lifecycle answer remains a typed wrong_result." + } + ] + } + }, + "openviking_context_trajectory_profile": { + "scenario_outcomes": [ + { + "scenario_id": "openviking-local-embed-setup", + "surface": "Docker local embedding setup", + "evidence_class": "live_baseline_only", + "result_type": "pass", + "openviking_status": "pass", + "elf_equivalent_status": "unsupported", + "elf_outcome": "not_tested", + "typed_blocker": null, + "evidence": "The pinned llama-cpp-python==0.3.28 CPU wheel path installed and OpenViking reached add_resource/find in Docker." + }, + { + "scenario_id": "openviking-evidence-bearing-retrieval-precondition", + "surface": "same-corpus evidence-bearing retrieval precondition", + "evidence_class": "live_baseline_only", + "result_type": "wrong_result", + "openviking_status": "wrong_result", + "elf_equivalent_status": "pass", + "elf_outcome": "elf_win", + "typed_blocker": "output_missed_expected_terms", + "evidence": "OpenViking add_resource/find returned resources but matched 0/3 expected evidence-term checks; this is a wrong_result smoke output, not a trajectory comparison." + }, + { + "scenario_id": "openviking-staged-retrieval-trajectory", + "surface": "staged retrieval trajectory", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "openviking_status": "not_encoded", + "elf_equivalent_status": "not_encoded", + "elf_outcome": "not_tested", + "typed_blocker": "needs_evidence_bearing_same_corpus_output", + "evidence": "No stage trajectory scoring is claimed until OpenViking returns evidence-bearing same-corpus output." + }, + { + "scenario_id": "openviking-hierarchy-selection", + "surface": "hierarchy selection", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "openviking_status": "not_encoded", + "elf_equivalent_status": "unsupported", + "elf_outcome": "not_tested", + "typed_blocker": "hierarchy_output_not_scored", + "evidence": "The viking:// hierarchy model remains a reference strength, but no real_world_job output scores hierarchy selection." + }, + { + "scenario_id": "openviking-recursive-context-expansion", + "surface": "recursive/context expansion", + "evidence_class": "research_gate", + "result_type": "not_encoded", + "openviking_status": "not_encoded", + "elf_equivalent_status": "not_encoded", + "elf_outcome": "not_tested", + "typed_blocker": "recursive_expansion_not_materialized", + "evidence": "Recursive/context expansion remains unmaterialized in the Docker adapter; no pass/fail quality claim is allowed." + }, + { + "scenario_id": "openviking-missed-expected-terms-evidence", + "surface": "typed failure evidence when expected terms are missed", + "evidence_class": "live_baseline_only", + "result_type": "wrong_result", + "openviking_status": "wrong_result", + "elf_equivalent_status": "pass", + "elf_outcome": "not_tested", + "typed_blocker": "retrieval_wrong_result", + "evidence": "The baseline report preserves the same missed expected terms as wrong_result instead of loosening evidence expectations or reporting setup failure; this row documents typed failure evidence and is not counted as a second comparative win." + } + ], + "win_tie_loss_summary": { + "elf_win": 1, + "tie": 0, + "elf_loss": 0, + "not_tested": 5 + } + }, + "claim_boundaries": [ + "ELF does not broadly beat qmd; it ties encoded retrieval and lifecycle correctness, keeps qmd query transparency as not_tested for comparative scoring, and leaves replayability not_tested.", + "qmd expansion, fusion, and rerank superiority remains not_tested because the current qmd paths use --no-rerank and do not score internals.", + "ELF does not beat OpenViking on context trajectory; OpenViking trajectory strengths remain not_tested behind a wrong_result same-corpus output precondition.", + "Research_gate records are follow-up gates, not pass evidence.", + "Missing equivalent surfaces are encoded as unsupported or not_encoded rather than fake losses." + ] +} diff --git a/docs/spec/real_world_agent_memory_benchmark_v1.md b/docs/spec/real_world_agent_memory_benchmark_v1.md index 115acc3f..fdc2f571 100644 --- a/docs/spec/real_world_agent_memory_benchmark_v1.md +++ b/docs/spec/real_world_agent_memory_benchmark_v1.md @@ -233,6 +233,12 @@ metadata, per-adapter records, and summary counters for: - capability coverage statuses; - real-world suite coverage statuses. +For `elf.real_world_external_adapter_report/v1`, `adapter_count` is the number of +adapter records in the loaded manifest. `external_project_count` is the number of +unique non-ELF project names represented by those records, not the number of non-ELF +adapter records. Multiple adapter records for the same external project MUST count as +one external project in this summary. + Adapter-pack issues SHOULD add new projects by appending adapter records to this manifest shape. They MUST NOT change these status meanings to make a project look better or worse.