From 5e5fa441a9e8df51a3a5f6d2a08738ada4cd2826 Mon Sep 17 00:00:00 2001 From: hyeokjun32 Date: Mon, 22 Jun 2026 13:56:53 +0900 Subject: [PATCH] Report AIGuard worker health trend evidence --- .../runtime_intelligence_gitlab_artifacts.md | 1 + .../edgeenv_runtime_regression_lab_handoff.md | 2 +- ...uard_runtime_operation_guard_analysis.json | 1744 +++++++++++++++-- .../bundle_manifest.json | 1 + .../edgeenv_lab_handoff_manifest.json | 13 +- ..._regression_with_orchestrator_context.json | 68 +- .../runtime_telemetry_history.json | 68 +- inferedgelab/report/runtime_intelligence.py | 59 + ...ck_runtime_intelligence_artifact_bundle.py | 15 +- ...ck_runtime_intelligence_bundle_manifest.py | 120 +- ...st_runtime_intelligence_bundle_manifest.py | 14 +- ...ntime_intelligence_evidence_chain_smoke.py | 45 +- 12 files changed, 1928 insertions(+), 222 deletions(-) diff --git a/docs/ci/runtime_intelligence_gitlab_artifacts.md b/docs/ci/runtime_intelligence_gitlab_artifacts.md index 5176bd2..f40fe9e 100644 --- a/docs/ci/runtime_intelligence_gitlab_artifacts.md +++ b/docs/ci/runtime_intelligence_gitlab_artifacts.md @@ -186,6 +186,7 @@ packaging runs. That marker set preserves `Runtime Intelligence Risk Summary`, `AIGuard operation risk rollup evidence`, `AIGuard operation timeline evidence`, `AIGuard scheduler fairness evidence`, +`AIGuard worker health trend evidence`, `AIGuard policy pressure evidence`, `AIGuard runtime operation anomalies`, `AIGuard remote dispatch event summary`, diff --git a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md index 131789f..bd29d69 100644 --- a/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md +++ b/docs/portfolio/edgeenv_runtime_regression_lab_handoff.md @@ -192,7 +192,7 @@ Expected Lab behavior: - The same gate requires EdgeEnv-preserved Orchestrator producer markers to carry `source_repository=InferEdgeOrchestrator`, `artifact_role=orchestrator-supplemental-operation-context`, and `producer_contract=inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1`. - When EdgeEnv preservation context is present, Lab renders `Lab EdgeEnv preservation context` with `lab_report_preservation_context_present=True`, `lab_preservation=present`, and `lab_context=present` so the Runtime Intelligence report gate and entrypoint evidence index use the same Lab-owned marker vocabulary. - When an EdgeEnv handoff manifest is provided, the bundle gate requires EdgeEnv-produced file keys, external AIGuard file keys, source repository mapping, artifact roles, producer contracts, and boundary flags to match Lab's Runtime Intelligence bundle contract. -- The same manifest gate requires `expected_report_markers` to preserve these exact Lab-owned report markers: `Runtime Intelligence Risk Summary`, `Runtime replay duration scope`, `Orchestrator operation feed context`, `EdgeEnv fixture matrix coverage`, `Reviewer operation quick scan`, `Orchestrator task event rollup`, `Lab EdgeEnv preservation context`, `AIGuard operation risk rollup evidence`, `AIGuard task event rollup evidence`, `AIGuard operation timeline evidence`, `AIGuard scheduler fairness evidence`, `AIGuard policy pressure evidence`, `AIGuard runtime operation anomalies`, `AIGuard remote dispatch event summary`, `AIGuard remote event summary consistency`, `Remote fallback starter evidence`, `lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback`, `AIGuard producer-lineage guard alignment`, and `Lab remains the final deployment decision owner.`. +- The same manifest gate requires `expected_report_markers` to preserve these exact Lab-owned report markers: `Runtime Intelligence Risk Summary`, `Runtime replay duration scope`, `Orchestrator operation feed context`, `EdgeEnv fixture matrix coverage`, `Reviewer operation quick scan`, `Orchestrator task event rollup`, `Lab EdgeEnv preservation context`, `AIGuard operation risk rollup evidence`, `AIGuard task event rollup evidence`, `AIGuard operation timeline evidence`, `AIGuard scheduler fairness evidence`, `AIGuard worker health trend evidence`, `AIGuard policy pressure evidence`, `AIGuard runtime operation anomalies`, `AIGuard remote dispatch event summary`, `AIGuard remote event summary consistency`, `Remote fallback starter evidence`, `lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback`, `AIGuard producer-lineage guard alignment`, and `Lab remains the final deployment decision owner.`. - Reviewer navigation can trace Orchestrator curated samples through AIGuard evidence into Lab report rows: `agent_scheduler_delay_sample.json` -> `scheduler_delay_pattern` -> `Reviewer operation quick scan` / `Orchestrator queue/deadline/fallback markers`, and `remote_fallback_recovery_sample.json` -> `remote_execution_recovered_by_fallback` -> `Remote fallback starter evidence`. Lab does not treat those curated samples as benchmark outputs, production retry proof, or deployment policy inputs. - The external AIGuard artifact gate also requires `runtime_queue_overload` raw context to preserve diff --git a/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json b/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json index faba683..a689379 100644 --- a/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json +++ b/examples/runtime_intelligence_chain/aiguard_runtime_operation_guard_analysis.json @@ -135,7 +135,8 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "orchestrator_mapping_hint_copy_candidate_context_to": "runtime_telemetry_context.candidate", @@ -150,14 +151,16 @@ ], "orchestrator_mapping_hint_aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "orchestrator_downstream_guard_alignment": { "declared_by": "orchestrator", "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -171,7 +174,8 @@ "orchestrator_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "orchestrator_guard_alignment_operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "orchestrator_guard_alignment_orchestrator_is_final_decision_owner": false, "orchestrator_guard_alignment_lab_is_final_decision_owner": true, @@ -251,19 +255,22 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "history_missing_orchestrator_downstream_guard_alignment": { "declared_by": "orchestrator", "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -277,7 +284,8 @@ "history_missing_orchestrator_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "history_missing_orchestrator_guard_alignment_operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "history_missing_orchestrator_guard_alignment_orchestrator_is_final_decision_owner": false, "history_missing_orchestrator_guard_alignment_lab_is_final_decision_owner": true, @@ -685,7 +693,8 @@ "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -699,7 +708,8 @@ "candidate_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "candidate_guard_alignment_operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "missing_expected": true, "missing_device_local_sources": [ @@ -726,7 +736,8 @@ "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -740,7 +751,8 @@ "missing_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "missing_guard_alignment_operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "missing_context_run_ids": [ "edgeenv-smoke-missing" @@ -1490,7 +1502,7 @@ { "type": "edgeenv_orchestrator_operation_timeline_summary", "metric_name": "orchestrator_operation_timeline_review_marker_count", - "observed_value": 6, + "observed_value": 7, "baseline_value": 0, "threshold": 1, "delta": null, @@ -1498,13 +1510,14 @@ "increase_factor": null, "severity": "medium", "status": "warning", - "explanation": "EdgeEnv preserved Orchestrator operation_timeline_summary with 6 deterministic review marker(s).", + "explanation": "EdgeEnv preserved Orchestrator operation_timeline_summary with 7 deterministic review marker(s).", "why_it_matters": "The preserved operation timeline compactly links queue pressure, latency wait, policy decisions, and affected tasks for Lab review without making AIGuard or Orchestrator the final deployment owner.", "suspected_causes": [ "scheduler_delay_context", "deadline_miss_context", "fallback_policy_context", - "queue_pressure_context" + "queue_pressure_context", + "worker_health_trend_context" ], "recommendation": "Review the preserved operation_timeline_summary in Lab alongside queue, latency, runtime event, and policy evidence; Lab remains the final deployment decision owner.", "raw_context": { @@ -1518,7 +1531,8 @@ "review_queue_pressure", "review_scheduler_delay", "review_deadline_miss", - "review_fallback_use" + "review_fallback_use", + "review_worker_health_trend" ], "affected_tasks": [ "vision_agent", @@ -1539,11 +1553,59 @@ "review_deadline_miss", "review_fallback_use", "affected_task_context", - "backlog_policy_reason" + "backlog_policy_reason", + "review_worker_health_trend" ], "decision_owner": "lab", "scheduler_owner": "orchestrator", - "not_a_deployment_decision": true + "not_a_deployment_decision": true, + "worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + } } } }, @@ -1640,150 +1702,73 @@ } }, { - "type": "edgeenv_orchestrator_policy_pressure_summary", - "metric_name": "orchestrator_policy_pressure_marker_count", + "type": "edgeenv_orchestrator_worker_health_trend", + "metric_name": "orchestrator_worker_health_trend_marker_count", "observed_value": 5, "baseline_value": 0, "threshold": 1, - "delta": 5, - "delta_pct": null, - "increase_factor": null, - "severity": "medium", - "status": "warning", - "explanation": "EdgeEnv preserved Orchestrator policy_pressure summary with 5 review markers.", - "why_it_matters": "Policy-pressure context explains which tasks were limited, protected, or sent to fallback under queue backlog pressure.", - "suspected_causes": [ - "policy_decision_context", - "queue_pressure_context", - "fallback_policy_context", - "load_shedding_context", - "scheduler_delay_context" - ], - "recommendation": "Review policy_pressure in Lab alongside operation timeline, task event rollup, and scheduler fairness evidence.", - "raw_context": { - "policy_pressure_summary": { - "summary": { - "schema_version": "inferedge-orchestrator-policy-pressure-summary-v1", - "role": "supplemental", - "decision_owner": "lab", - "scheduler_owner": "orchestrator", - "not_a_deployment_decision": true, - "decision_count": 2, - "decision_reason_counts": { - "queue_backlog_threshold_exceeded": 2 - }, - "limited_tasks": [ - "vision_agent", - "voice_command_agent" - ], - "protected_tasks": [ - "safety_monitor_agent" - ], - "fallback_tasks": [ - "voice_command_agent" - ], - "fallback_decision_count": 1, - "backlog_thresholds": [ - 3 - ], - "max_total_backlog_before": 7, - "max_backlog_over_threshold": 4, - "pressure_markers": [ - "policy_decision_present", - "backlog_exceeded_threshold", - "fallback_policy_used", - "workload_limited_by_policy", - "scheduler_delay_present" - ], - "interpretation": "Scheduler policy pressure preserved as Lab review context only.", - "first_read": "review_policy_pressure_context" - }, - "marker_count": 5, - "decision_count": 2, - "decision_reason_counts": { - "queue_backlog_threshold_exceeded": 2 - }, - "limited_tasks": [ - "vision_agent", - "voice_command_agent" - ], - "protected_tasks": [ - "safety_monitor_agent" - ], - "fallback_tasks": [ - "voice_command_agent" - ], - "max_backlog_over_threshold": 4, - "decision_owner": "lab", - "scheduler_owner": "orchestrator", - "not_a_deployment_decision": true, - "boundary_markers_valid": true - } - } - }, - { - "type": "runtime_history_seed_run_config_traceability", - "metric_name": "runtime_history_seed_run_config_runs", - "observed_value": 2, - "baseline_value": 2, - "threshold": 2, - "delta": 0, - "delta_pct": null, - "increase_factor": null, - "severity": "low", - "status": "passed", - "explanation": "Runtime history seed run_config markers are preserved for 2 of 2 expected Runtime history seed runs.", - "why_it_matters": "Runtime history seed run_config markers make EdgeEnv replay and comparability context traceable without making AIGuard or EdgeEnv the deployment decision owner.", - "suspected_causes": [], - "recommendation": "Preserve Runtime history seed run_config markers in the Lab report as traceability evidence.", - "raw_context": { - "history_seed_run_config": { - "marker_labels": [ - "baseline/candidate=shape=1x640x640, input_mode=dummy, input_preprocess=none, power_mode=unknown, jetson_clocks=unknown, warmup=1, runs=10" - ], - "history_seed_run_config_runs": 2, - "baseline_run_config_present": true, - "candidate_run_config_present": true, - "registry_owner": "edgeenv", - "decision_owner": "lab" - } - } - }, - { - "type": "runtime_thermal_instability", - "metric_name": "candidate_max_temperature_c", - "observed_value": 78.5, - "baseline_value": null, - "threshold": 70.0, "delta": null, "delta_pct": null, "increase_factor": null, - "severity": "medium", + "severity": "high", "status": "warning", - "explanation": "candidate_max_temperature_c observed value is 78.5. Threshold is 70. This runtime_thermal_instability evidence should be reviewed before deployment.", - "why_it_matters": "Thermal pressure or throttling in EdgeEnv telemetry can explain runtime latency drift and reduce sustained deployment reliability.", + "explanation": "EdgeEnv preserved Orchestrator worker_health_trend with 5 deterministic review marker(s).", + "why_it_matters": "The preserved worker-health trend links degraded or constrained worker context to Orchestrator scheduling evidence while keeping Lab as the final deployment decision owner.", "suspected_causes": [ - "thermal_pressure", - "thermal_throttling", - "power_mode_or_cooling_constraint" + "worker_health_degradation_context", + "worker_health_constraint_context", + "worker_health_state_trend_context" ], - "recommendation": "Review EdgeEnv telemetry history, power mode, cooling, and sustained run conditions before treating runtime regression as stable.", + "recommendation": "Review worker_health_trend in Lab alongside scheduler fairness, queue pressure, and stale-drop evidence before treating the operation profile as stable.", "raw_context": { "edgeenv_regression": { "baseline_run_id": "edgeenv-smoke-baseline", "candidate_run_id": "edgeenv-smoke-candidate", "comparable": true, "mode": "same-condition", - "regression_detected": false, - "regression_type": "operation-context", - "severity": "medium", + "regression_detected": true, + "regression_type": "mixed", + "severity": "high", "recommendation": "review_required", - "mean_delta_pct": null, - "p95_delta_pct": null, - "p99_delta_pct": null, - "fps_delta_pct": null, - "memory_peak_delta_pct": null, - "triggered_thresholds": [], + "mean_delta_pct": 18.0, + "p95_delta_pct": 10.0, + "p99_delta_pct": 32.0, + "fps_delta_pct": -22.0, + "memory_peak_delta_pct": 40.0, + "triggered_thresholds": [ + { + "metric": "latency_mean_ms", + "name": "mean_latency_review", + "observed": 18.0, + "severity": "medium", + "threshold": 15.0, + "type": "latency" + }, + { + "metric": "latency_p99_ms", + "name": "p99_latency_high", + "observed": 32.0, + "severity": "high", + "threshold": 25.0, + "type": "latency" + }, + { + "metric": "throughput_fps", + "name": "fps_drop_review", + "observed": -22.0, + "severity": "medium", + "threshold": -20.0, + "type": "latency" + }, + { + "metric": "memory_peak_mb", + "name": "memory_peak_warning", + "observed": 40.0, + "severity": "warning", + "threshold": 30.0, + "type": "resource" + } + ], "runtime_telemetry_context_present": true, "runtime_telemetry_source": "result_artifacts+runtime_telemetry_history", "runtime_telemetry_history_schema_version": "edgeenv.runtime-telemetry-history.v1", @@ -1792,6 +1777,26 @@ "history_telemetry_runs": 2.0, "history_telemetry_seed_runs": 2.0, "history_telemetry_seed_run_config_runs": 2.0, + "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", + "candidate_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", + "baseline_runtime_telemetry_history_seed_registry_owner": "edgeenv", + "candidate_runtime_telemetry_history_seed_registry_owner": "edgeenv", + "baseline_runtime_telemetry_history_seed_decision_owner": "lab", + "candidate_runtime_telemetry_history_seed_decision_owner": "lab", + "candidate_runtime_telemetry_history_seed_production_monitoring": false, + "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, + "candidate_runtime_telemetry_history_seed_point_count": 1.0, + "runtime_telemetry_history_seed_run_config_marker_fields": [ + "input_mode", + "input_preprocess", + "power_mode", + "jetson_clocks", + "warmup", + "runs" + ], + "runtime_telemetry_history_seed_run_config_marker_labels": [ + "baseline/candidate=shape=1x640x640, input_mode=dummy, input_preprocess=none, power_mode=unknown, jetson_clocks=unknown, warmup=1, runs=10" + ], "baseline_runtime_telemetry_history_seed_run_config": { "batch": 1, "height": 640, @@ -1804,6 +1809,15 @@ "power_mode": "unknown", "jetson_clocks": "unknown" }, + "baseline_runtime_telemetry_history_seed_run_config_markers": { + "shape": "1x640x640", + "input_mode": "dummy", + "input_preprocess": "none", + "power_mode": "unknown", + "jetson_clocks": "unknown", + "warmup": 1, + "runs": 10 + }, "candidate_runtime_telemetry_history_seed_run_config": { "batch": 1, "height": 640, @@ -1816,48 +1830,1170 @@ "power_mode": "unknown", "jetson_clocks": "unknown" }, + "candidate_runtime_telemetry_history_seed_run_config_markers": { + "shape": "1x640x640", + "input_mode": "dummy", + "input_preprocess": "none", + "power_mode": "unknown", + "jetson_clocks": "unknown", + "warmup": 1, + "runs": 10 + }, "candidate_runtime_telemetry_history_seed_run_config_present": true, - "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", - "candidate_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", - "baseline_runtime_telemetry_history_seed_registry_owner": "edgeenv", - "candidate_runtime_telemetry_history_seed_registry_owner": "edgeenv", - "baseline_runtime_telemetry_history_seed_decision_owner": "lab", - "candidate_runtime_telemetry_history_seed_decision_owner": "lab", - "candidate_runtime_telemetry_history_seed_production_monitoring": false, - "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, - "candidate_runtime_telemetry_history_seed_point_count": 1.0, "history_missing_telemetry_runs": 1.0, - "telemetry_coverage_source": "history_telemetry_coverage", - "history_telemetry_coverage_missing_field_run_count": 1.0, - "history_telemetry_coverage_missing_field_runs": [ - { - "run_id": "edgeenv-smoke-candidate", - "missing_fields": [ - "queue_depth" - ], - "missing_field_count": 1, - "missing_telemetry_is_failure": false - } - ], - "history_telemetry_coverage_run_summaries_present": true, - "baseline_telemetry_present": true, - "candidate_telemetry_present": true, - "baseline_history_entry_present": true, - "candidate_history_entry_present": true, - "baseline_telemetry_coverage_ratio": 1.0, - "candidate_telemetry_coverage_ratio": 0.666667, - "baseline_telemetry_coverage_missing_fields": [], - "candidate_telemetry_coverage_missing_fields": [ - "queue_depth" + "history_missing_orchestrator_context_count": 1.0, + "history_missing_orchestrator_context_run_ids": [ + "edgeenv-smoke-missing" ], - "telemetry_coverage_missing_field_count": 1.0, - "baseline_missing_telemetry_is_failure": false, - "candidate_missing_telemetry_is_failure": false, - "baseline_execution_sequence_id": 1.0, - "candidate_execution_sequence_id": 2.0, - "execution_sequence_order_valid": true, - "baseline_orchestrator_context_present": false, - "candidate_orchestrator_context_present": true, + "history_missing_orchestrator_contexts": [ + { + "schema_version": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "role": "orchestrator_operation_context_for_edgeenv", + "source_repository": "InferEdgeOrchestrator", + "artifact_role": "orchestrator-supplemental-operation-context", + "producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "source": "orchestration_summary", + "run_id": "edgeenv-smoke-missing", + "not_a_regression_judgement": true, + "not_a_comparability_gate": true, + "decision_owner": "lab", + "regression_owner": "edgeenv", + "candidate_context": { + "run_id": "edgeenv-smoke-missing", + "telemetry_source": "inferedge_orchestrator_operation_summary", + "queue_depth": 4, + "operation": { + "queue_depth": 4, + "deadline_missed_count": 1, + "fallback_count": 0 + }, + "resource": { + "source": "runtime_health_snapshot", + "gpu_temperature": 72.0, + "throttling_detected": false + }, + "producer": { + "operation_context_role": "supplemental", + "producer_sources": [ + "device_local_cli_override", + "orchestration_summary" + ], + "device_local_producer_sources": [ + "device_local_cli_override" + ], + "producer_sources_by_task": { + "vision_agent": [ + "device_local_cli_override" + ] + }, + "producer_stage_by_task": { + "vision_agent": "device_local_starter" + }, + "producer_event_count": 4, + "device_local_event_count": 2, + "device_local_task_count": 1 + } + }, + "edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ] + }, + "downstream_guard_alignment": { + "declared_by": "orchestrator", + "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", + "operation_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "validated_by": [ + "edgeenv runs telemetry inspect-history", + "inferedge-aiguard reason-edgeenv-regression", + "inferedgelab runtime-intelligence bundle manifest gate" + ], + "orchestrator_is_final_decision_owner": false, + "lab_is_final_decision_owner": true + }, + "remote_runtime_event_summary": { + "event_count": 4, + "runtime_event_count": 4, + "production_remote_execution": false, + "evidence_role": "remote_dispatch_runtime_event_compact_summary", + "operation_boundary": "remote dispatch starter evidence only" + } + } + ], + "history_missing_orchestrator_source_repository": "InferEdgeOrchestrator", + "history_missing_orchestrator_artifact_role": "orchestrator-supplemental-operation-context", + "history_missing_orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "history_missing_orchestrator_candidate_context_telemetry_source": "inferedge_orchestrator_operation_summary", + "history_missing_orchestrator_candidate_context_producer": { + "operation_context_role": "supplemental", + "producer_sources": [ + "device_local_cli_override", + "orchestration_summary" + ], + "device_local_producer_sources": [ + "device_local_cli_override" + ], + "producer_sources_by_task": { + "vision_agent": [ + "device_local_cli_override" + ] + }, + "producer_stage_by_task": { + "vision_agent": "device_local_starter" + }, + "producer_event_count": 4, + "device_local_event_count": 2, + "device_local_task_count": 1 + }, + "history_missing_orchestrator_candidate_producer_sources": [ + "device_local_cli_override", + "orchestration_summary" + ], + "history_missing_orchestrator_candidate_device_local_producer_sources": [ + "device_local_cli_override" + ], + "history_missing_orchestrator_candidate_producer_sources_by_task": { + "vision_agent": [ + "device_local_cli_override" + ] + }, + "history_missing_orchestrator_candidate_producer_stage_by_task": { + "vision_agent": "device_local_starter" + }, + "history_missing_orchestrator_candidate_producer_event_count": 4.0, + "history_missing_orchestrator_candidate_device_local_event_count": 2.0, + "history_missing_orchestrator_candidate_device_local_task_count": 1.0, + "history_missing_orchestrator_candidate_operation_context_role": "supplemental", + "history_missing_orchestrator_remote_runtime_event_summary": { + "event_count": 4, + "runtime_event_count": 4, + "production_remote_execution": false, + "evidence_role": "remote_dispatch_runtime_event_compact_summary", + "operation_boundary": "remote dispatch starter evidence only" + }, + "history_missing_orchestrator_remote_runtime_event_summary_present": true, + "history_missing_orchestrator_remote_runtime_event_summary_evidence_role": "remote_dispatch_runtime_event_compact_summary", + "history_missing_orchestrator_remote_runtime_event_summary_operation_boundary": "remote dispatch starter evidence only", + "history_missing_orchestrator_remote_runtime_event_summary_production_remote_execution": false, + "history_missing_orchestrator_operation_risk_summary": {}, + "history_missing_orchestrator_operation_risk_summary_present": false, + "history_missing_orchestrator_operation_risk_summary_evidence_role": null, + "history_missing_orchestrator_operation_risk_summary_decision_owner": null, + "history_missing_orchestrator_operation_risk_summary_scheduler_owner": null, + "history_missing_orchestrator_operation_risk_summary_not_a_deployment_decision": null, + "history_missing_orchestrator_edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ] + }, + "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "history_missing_orchestrator_downstream_guard_alignment": { + "declared_by": "orchestrator", + "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", + "operation_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "validated_by": [ + "edgeenv runs telemetry inspect-history", + "inferedge-aiguard reason-edgeenv-regression", + "inferedgelab runtime-intelligence bundle manifest gate" + ], + "orchestrator_is_final_decision_owner": false, + "lab_is_final_decision_owner": true + }, + "history_missing_orchestrator_guard_alignment_declared_by": "orchestrator", + "history_missing_orchestrator_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", + "history_missing_orchestrator_guard_alignment_operation_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "history_missing_orchestrator_guard_alignment_orchestrator_is_final_decision_owner": false, + "history_missing_orchestrator_guard_alignment_lab_is_final_decision_owner": true, + "telemetry_coverage_source": "history_telemetry_coverage", + "history_telemetry_coverage_missing_field_run_count": 1.0, + "history_telemetry_coverage_missing_field_runs": [ + { + "run_id": "edgeenv-smoke-candidate", + "missing_fields": [ + "queue_depth" + ], + "missing_field_count": 1, + "missing_telemetry_is_failure": false + } + ], + "history_telemetry_coverage_run_summaries_present": true, + "baseline_telemetry_present": true, + "candidate_telemetry_present": true, + "baseline_history_entry_present": true, + "candidate_history_entry_present": true, + "baseline_telemetry_coverage_ratio": 1.0, + "candidate_telemetry_coverage_ratio": 0.666667, + "baseline_telemetry_coverage_missing_fields": [], + "candidate_telemetry_coverage_missing_fields": [ + "queue_depth" + ], + "telemetry_coverage_missing_field_count": 1.0, + "baseline_missing_telemetry_is_failure": false, + "candidate_missing_telemetry_is_failure": false, + "baseline_execution_sequence_id": 1.0, + "candidate_execution_sequence_id": 2.0, + "execution_sequence_order_valid": true, + "baseline_orchestrator_context_present": false, + "candidate_orchestrator_context_present": true, + "orchestrator_source_repository": "InferEdgeOrchestrator", + "orchestrator_artifact_role": "orchestrator-supplemental-operation-context", + "orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", + "orchestrator_candidate_context_telemetry_source": "inferedge_orchestrator_operation_summary", + "orchestrator_candidate_context_producer": { + "operation_context_role": "supplemental", + "frames": 96, + "duration_class": "short_96_frame_class", + "duration_label": "short 96-frame-class replay (96 frames)", + "duration_source": "entrypoint_requested_frames", + "duration_scope_label": "source=entrypoint_requested_frames, label=short 96-frame-class replay (96 frames), class=short_96_frame_class, frames=96", + "producer_sources": [ + "device_local_cli_override", + "orchestration_summary" + ], + "device_local_producer_sources": [ + "device_local_cli_override" + ], + "producer_sources_by_task": { + "vision_agent": [ + "device_local_cli_override" + ] + }, + "producer_stage_by_task": { + "vision_agent": "device_local_starter" + }, + "producer_event_count": 4, + "device_local_event_count": 2, + "device_local_task_count": 1 + }, + "orchestrator_candidate_producer_sources": [ + "device_local_cli_override", + "orchestration_summary" + ], + "orchestrator_candidate_device_local_producer_sources": [ + "device_local_cli_override" + ], + "orchestrator_candidate_producer_sources_by_task": { + "vision_agent": [ + "device_local_cli_override" + ] + }, + "orchestrator_candidate_producer_stage_by_task": { + "vision_agent": "device_local_starter" + }, + "orchestrator_candidate_producer_event_count": 4.0, + "orchestrator_candidate_device_local_event_count": 2.0, + "orchestrator_candidate_device_local_task_count": 1.0, + "orchestrator_candidate_operation_context_role": "supplemental", + "orchestrator_remote_runtime_event_summary": { + "event_count": 4, + "runtime_event_count": 4, + "production_remote_execution": false, + "evidence_role": "remote_dispatch_runtime_event_compact_summary", + "operation_boundary": "remote dispatch starter evidence only" + }, + "orchestrator_remote_runtime_event_summary_present": true, + "orchestrator_remote_runtime_event_summary_evidence_role": "remote_dispatch_runtime_event_compact_summary", + "orchestrator_remote_runtime_event_summary_operation_boundary": "remote dispatch starter evidence only", + "orchestrator_remote_runtime_event_summary_production_remote_execution": false, + "orchestrator_operation_risk_summary": { + "schema_version": "inferedge-entrypoint-operation-risk-summary-v1", + "evidence_role": "derived_navigation_context", + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "queue_pressure_reason": "queue_backlog_threshold_exceeded", + "max_pressure_task": "vision_agent", + "primary_health_reason": "worker_health_degraded", + "degraded_worker_ids": [ + "vision_agent" + ], + "device_local_event_count": 15, + "producer_event_count": 7, + "not_a_deployment_decision": true + }, + "orchestrator_operation_risk_summary_present": true, + "orchestrator_operation_risk_summary_schema_version": "inferedge-entrypoint-operation-risk-summary-v1", + "orchestrator_operation_risk_summary_evidence_role": "derived_navigation_context", + "orchestrator_operation_risk_summary_decision_owner": "lab", + "orchestrator_operation_risk_summary_scheduler_owner": "orchestrator", + "orchestrator_operation_risk_summary_not_a_deployment_decision": true, + "orchestrator_operation_risk_summary_queue_pressure_reason": "queue_backlog_threshold_exceeded", + "orchestrator_operation_risk_summary_max_pressure_task": "vision_agent", + "orchestrator_operation_risk_summary_primary_health_reason": "worker_health_degraded", + "orchestrator_operation_risk_summary_degraded_worker_ids": [ + "vision_agent" + ], + "orchestrator_operation_risk_summary_device_local_event_count": 15.0, + "orchestrator_operation_risk_summary_producer_event_count": 7.0, + "orchestrator_operation_risk_rollup": { + "schema_version": "inferedge-orchestrator-operation-risk-rollup-v1", + "operation_context_role": "supplemental", + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "not_a_deployment_decision": true, + "risk_level": "review", + "first_read": "review_operation_risk_context", + "primary_reasons": [ + "queue_pressure_overloaded", + "scheduler_delay_present", + "deadline_missed_present", + "fallback_used" + ], + "affected_tasks": { + "deadline_missed": [ + "vision_agent" + ], + "fallback": [ + "voice_command_agent" + ], + "scheduler_delay": [ + "vision_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [] + }, + "max_total_queue_depth": 7, + "overload_backlog_threshold": 3, + "queue_pressure_state": "overloaded", + "queue_pressure_reason": "queue_backlog_threshold_exceeded", + "deadline_missed_count": 2, + "fallback_count": 1, + "dropped_count": 1, + "policy_decision_count": 2, + "scheduler_delay_event_count": 1, + "resource_snapshot_count": 0 + }, + "orchestrator_operation_risk_rollup_present": true, + "orchestrator_operation_risk_rollup_schema_version": "inferedge-orchestrator-operation-risk-rollup-v1", + "orchestrator_operation_risk_rollup_operation_context_role": "supplemental", + "orchestrator_operation_risk_rollup_decision_owner": "lab", + "orchestrator_operation_risk_rollup_scheduler_owner": "orchestrator", + "orchestrator_operation_risk_rollup_not_a_deployment_decision": true, + "orchestrator_operation_risk_rollup_risk_level": "review", + "orchestrator_operation_risk_rollup_first_read": "review_operation_risk_context", + "orchestrator_operation_risk_rollup_primary_reasons": [ + "queue_pressure_overloaded", + "scheduler_delay_present", + "deadline_missed_present", + "fallback_used" + ], + "orchestrator_operation_risk_rollup_affected_tasks": { + "deadline_missed": [ + "vision_agent" + ], + "fallback": [ + "voice_command_agent" + ], + "scheduler_delay": [ + "vision_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [] + }, + "orchestrator_operation_risk_rollup_affected_deadline_missed_tasks": [ + "vision_agent" + ], + "orchestrator_operation_risk_rollup_affected_fallback_tasks": [ + "voice_command_agent" + ], + "orchestrator_operation_risk_rollup_affected_scheduler_delay_tasks": [ + "vision_agent" + ], + "orchestrator_operation_risk_rollup_affected_degraded_tasks": [ + "vision_agent" + ], + "orchestrator_operation_risk_rollup_affected_constrained_tasks": [], + "orchestrator_operation_risk_rollup_queue_pressure_state": "overloaded", + "orchestrator_operation_risk_rollup_queue_pressure_reason": "queue_backlog_threshold_exceeded", + "orchestrator_operation_risk_rollup_max_total_queue_depth": 7.0, + "orchestrator_operation_risk_rollup_deadline_missed_count": 2.0, + "orchestrator_operation_risk_rollup_fallback_count": 1.0, + "orchestrator_operation_risk_rollup_drop_count": 1.0, + "orchestrator_operation_risk_rollup_scheduler_delay_event_count": 1.0, + "orchestrator_operation_risk_rollup_policy_decision_count": 2.0, + "orchestrator_operation_risk_rollup_resource_snapshot_count": 0.0, + "orchestrator_latency_budget_protection": {}, + "orchestrator_latency_budget_protection_present": false, + "orchestrator_latency_budget_protection_schema_version": null, + "orchestrator_latency_budget_protection_evidence_role": null, + "orchestrator_latency_budget_protection_decision_owner": null, + "orchestrator_latency_budget_protection_scheduler_owner": null, + "orchestrator_latency_budget_protection_regression_owner": null, + "orchestrator_latency_budget_protection_not_a_deployment_decision": null, + "orchestrator_latency_budget_protection_protected_tasks": [], + "orchestrator_latency_budget_protection_risk_tasks": [], + "orchestrator_latency_budget_protection_risk_reasons": [], + "orchestrator_latency_budget_protection_per_task_budget_context": {}, + "orchestrator_operation_timeline_summary": { + "schema_version": "inferedge-orchestrator-operation-timeline-summary-v1", + "source": "queue_depth_timeline+latency_timeline+policy_decision_log+runtime_event_summary", + "sample_counts": { + "queue_depth": 2, + "latency": 2, + "policy_decision": 2, + "runtime_event": 15 + }, + "queue": { + "max_total_queue_depth": 7, + "average_total_queue_depth": 4.5, + "overload_backlog_threshold": 3, + "pressure_state": "overloaded", + "pressure_reason": "queue_backlog_threshold_exceeded", + "max_pressure_task": "vision_agent", + "max_queue_depth_by_task": { + "vision_agent": 7, + "voice_command_agent": 4 + } + }, + "latency": { + "sample_count": 2, + "max_latency_ms": 72.0, + "max_queue_wait_ms": 15.0, + "max_queue_wait_ms_by_task": { + "vision_agent": 15.0 + }, + "tasks_with_deadline_miss": [ + "vision_agent" + ] + }, + "policy": { + "decision_count": 2, + "decision_reasons": [ + "queue_backlog_threshold_exceeded" + ], + "first_decision": { + "decision_reason": "queue_backlog_threshold_exceeded" + }, + "latest_decision": { + "decision_reason": "queue_backlog_threshold_exceeded" + } + }, + "affected_tasks": { + "deadline_missed": [ + "vision_agent" + ], + "fallback": [ + "voice_command_agent" + ], + "scheduler_delay": [ + "vision_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "review_hints": [ + "review_queue_pressure", + "review_scheduler_delay", + "review_deadline_miss", + "review_fallback_use", + "review_worker_health_trend" + ], + "worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + } + }, + "orchestrator_operation_timeline_summary_present": true, + "orchestrator_operation_timeline_summary_schema_version": "inferedge-orchestrator-operation-timeline-summary-v1", + "orchestrator_operation_timeline_review_hints": [ + "review_queue_pressure", + "review_scheduler_delay", + "review_deadline_miss", + "review_fallback_use", + "review_worker_health_trend" + ], + "orchestrator_operation_timeline_queue_pressure_state": "overloaded", + "orchestrator_operation_timeline_queue_pressure_reason": "queue_backlog_threshold_exceeded", + "orchestrator_operation_timeline_max_pressure_task": "vision_agent", + "orchestrator_operation_timeline_max_queue_wait_ms": 15.0, + "orchestrator_operation_timeline_max_latency_ms": 72.0, + "orchestrator_operation_timeline_policy_decision_count": 2.0, + "orchestrator_operation_timeline_policy_decision_reasons": [ + "queue_backlog_threshold_exceeded" + ], + "orchestrator_policy_pressure_summary": {}, + "orchestrator_policy_pressure_summary_present": false, + "orchestrator_policy_pressure_summary_schema_version": null, + "orchestrator_policy_pressure_decision_count": null, + "orchestrator_policy_pressure_decision_reason_counts": {}, + "orchestrator_policy_pressure_limited_tasks": [], + "orchestrator_policy_pressure_protected_tasks": [], + "orchestrator_policy_pressure_fallback_tasks": [], + "orchestrator_policy_pressure_markers": [], + "orchestrator_policy_pressure_marker_count": 0.0, + "orchestrator_policy_pressure_max_backlog_over_threshold": null, + "orchestrator_policy_pressure_decision_owner": null, + "orchestrator_policy_pressure_scheduler_owner": null, + "orchestrator_policy_pressure_not_a_deployment_decision": null, + "orchestrator_operation_timeline_affected_deadline_missed_tasks": [ + "vision_agent" + ], + "orchestrator_operation_timeline_affected_fallback_tasks": [ + "voice_command_agent" + ], + "orchestrator_operation_timeline_affected_scheduler_delay_tasks": [ + "vision_agent" + ], + "orchestrator_operation_timeline_affected_degraded_tasks": [ + "vision_agent" + ], + "orchestrator_operation_timeline_affected_constrained_tasks": [ + "voice_command_agent" + ], + "orchestrator_scheduler_fairness_summary": {}, + "orchestrator_scheduler_fairness_summary_present": false, + "orchestrator_scheduler_fairness_summary_schema_version": null, + "orchestrator_scheduler_fairness_protected_high_priority_tasks": [], + "orchestrator_scheduler_fairness_starvation_risk_tasks": [], + "orchestrator_scheduler_fairness_scheduler_delay_tasks": [], + "orchestrator_scheduler_fairness_degraded_tasks": [], + "orchestrator_scheduler_fairness_task_context": {}, + "orchestrator_scheduler_fairness_starvation_risk_count": 0.0, + "orchestrator_scheduler_fairness_decision_owner": null, + "orchestrator_scheduler_fairness_scheduler_owner": null, + "orchestrator_scheduler_fairness_not_a_deployment_decision": null, + "orchestrator_worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + }, + "orchestrator_worker_health_trend_present": true, + "orchestrator_worker_health_trend_schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "orchestrator_worker_health_trend_operation_context_role": "supplemental", + "orchestrator_worker_health_trend_scheduler_owner": "orchestrator", + "orchestrator_worker_health_trend_decision_owner": "lab", + "orchestrator_worker_health_trend_not_a_deployment_decision": true, + "orchestrator_worker_health_trend_health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "orchestrator_worker_health_trend_tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "orchestrator_worker_health_trend_task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "orchestrator_worker_health_trend_degraded_workers": [ + "vision_agent" + ], + "orchestrator_worker_health_trend_constrained_workers": [ + "voice_command_agent" + ], + "orchestrator_worker_health_trend_review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "orchestrator_operation_timeline_affected_stale_drop_tasks": [], + "orchestrator_stale_drop_summary": {}, + "orchestrator_stale_drop_summary_present": false, + "orchestrator_stale_drop_summary_schema_version": null, + "orchestrator_stale_drop_count": 0.0, + "orchestrator_stale_drop_total_count": 0.0, + "orchestrator_stale_drop_rate": 0.0, + "orchestrator_stale_drop_reasons": {}, + "orchestrator_stale_drop_reason_classes": [], + "orchestrator_tasks_with_stale_drop": [], + "orchestrator_latest_stale_drop_event": {}, + "orchestrator_stale_drop_decision_owner": null, + "orchestrator_stale_drop_scheduler_owner": null, + "orchestrator_stale_drop_not_a_deployment_decision": null, + "orchestrator_runtime_task_event_summary": { + "vision_agent": { + "event_count": 9, + "event_type_counts": { + "execution": 4, + "schedule": 4, + "policy_decision": 1 + }, + "reason_counts": { + "deadline_missed": 1, + "queue_backlog_threshold_exceeded": 1 + }, + "policy_decision_reason_counts": { + "queue_backlog_threshold_exceeded": 1 + }, + "drop_reason_counts": {}, + "deadline_missed_count": 1, + "fallback_decision_count": 0, + "scheduler_delay_event_count": 1, + "max_scheduler_delay_cycles": 3, + "max_queue_wait_ms": 15.0, + "latest_event_index": 42, + "latest_event_type": "execution" + }, + "voice_command_agent": { + "event_count": 6, + "event_type_counts": { + "drop": 1, + "policy_decision": 1, + "schedule": 2, + "execution": 2 + }, + "reason_counts": { + "queue_backlog_threshold_exceeded": 1, + "load_shedding_backlog_threshold_exceeded": 1 + }, + "policy_decision_reason_counts": { + "queue_backlog_threshold_exceeded": 1 + }, + "drop_reason_counts": { + "load_shedding_backlog_threshold_exceeded": 1 + }, + "deadline_missed_count": 0, + "fallback_decision_count": 1, + "scheduler_delay_event_count": 0, + "max_scheduler_delay_cycles": 0, + "max_queue_wait_ms": 0.0, + "latest_event_index": 39, + "latest_event_type": "policy_decision" + } + }, + "orchestrator_runtime_task_event_summary_present": true, + "orchestrator_tasks_with_deadline_miss": [ + "vision_agent" + ], + "orchestrator_tasks_with_fallback": [ + "voice_command_agent" + ], + "orchestrator_tasks_with_scheduler_delay": [ + "vision_agent" + ], + "orchestrator_edgeenv_mapping_hint": { + "copy_candidate_context_to": "runtime_telemetry_context.candidate", + "operation_context_role": "supplemental", + "coverage_summary_owner": "edgeenv", + "coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ] + }, + "orchestrator_mapping_hint_copy_candidate_context_to": "runtime_telemetry_context.candidate", + "orchestrator_mapping_hint_operation_context_role": "supplemental", + "orchestrator_mapping_hint_coverage_summary_owner": "edgeenv", + "orchestrator_mapping_hint_coverage_summary_path": "runtime_telemetry_context.history.telemetry_coverage", + "orchestrator_mapping_hint_candidate_context_required_fields": [ + "run_id", + "telemetry_source", + "operation", + "resource" + ], + "orchestrator_mapping_hint_aiguard_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "orchestrator_downstream_guard_alignment": { + "declared_by": "orchestrator", + "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", + "operation_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "validated_by": [ + "edgeenv runs telemetry inspect-history", + "inferedge-aiguard reason-edgeenv-regression", + "inferedgelab runtime-intelligence bundle manifest gate" + ], + "orchestrator_is_final_decision_owner": false, + "lab_is_final_decision_owner": true + }, + "orchestrator_guard_alignment_declared_by": "orchestrator", + "orchestrator_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", + "orchestrator_guard_alignment_operation_evidence_candidates": [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" + ], + "orchestrator_guard_alignment_orchestrator_is_final_decision_owner": false, + "orchestrator_guard_alignment_lab_is_final_decision_owner": true, + "baseline_max_temperature_c": null, + "candidate_max_temperature_c": 78.5, + "baseline_throttling_detected": null, + "candidate_throttling_detected": true, + "baseline_queue_depth": null, + "candidate_queue_depth": 7.0, + "orchestrator_candidate_operation_max_total_queue_depth": 7.0, + "evidence_gap_count": 1.0, + "evidence_gaps": [] + }, + "worker_health_trend": { + "summary": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + }, + "boundary_markers_valid": true, + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "review_markers": [ + "degraded_worker_context", + "constrained_worker_context", + "health_state_count_context", + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "not_a_deployment_decision": true + } + } + }, + { + "type": "edgeenv_orchestrator_policy_pressure_summary", + "metric_name": "orchestrator_policy_pressure_marker_count", + "observed_value": 5, + "baseline_value": 0, + "threshold": 1, + "delta": 5, + "delta_pct": null, + "increase_factor": null, + "severity": "medium", + "status": "warning", + "explanation": "EdgeEnv preserved Orchestrator policy_pressure summary with 5 review markers.", + "why_it_matters": "Policy-pressure context explains which tasks were limited, protected, or sent to fallback under queue backlog pressure.", + "suspected_causes": [ + "policy_decision_context", + "queue_pressure_context", + "fallback_policy_context", + "load_shedding_context", + "scheduler_delay_context" + ], + "recommendation": "Review policy_pressure in Lab alongside operation timeline, task event rollup, and scheduler fairness evidence.", + "raw_context": { + "policy_pressure_summary": { + "summary": { + "schema_version": "inferedge-orchestrator-policy-pressure-summary-v1", + "role": "supplemental", + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "not_a_deployment_decision": true, + "decision_count": 2, + "decision_reason_counts": { + "queue_backlog_threshold_exceeded": 2 + }, + "limited_tasks": [ + "vision_agent", + "voice_command_agent" + ], + "protected_tasks": [ + "safety_monitor_agent" + ], + "fallback_tasks": [ + "voice_command_agent" + ], + "fallback_decision_count": 1, + "backlog_thresholds": [ + 3 + ], + "max_total_backlog_before": 7, + "max_backlog_over_threshold": 4, + "pressure_markers": [ + "policy_decision_present", + "backlog_exceeded_threshold", + "fallback_policy_used", + "workload_limited_by_policy", + "scheduler_delay_present" + ], + "interpretation": "Scheduler policy pressure preserved as Lab review context only.", + "first_read": "review_policy_pressure_context" + }, + "marker_count": 5, + "decision_count": 2, + "decision_reason_counts": { + "queue_backlog_threshold_exceeded": 2 + }, + "limited_tasks": [ + "vision_agent", + "voice_command_agent" + ], + "protected_tasks": [ + "safety_monitor_agent" + ], + "fallback_tasks": [ + "voice_command_agent" + ], + "max_backlog_over_threshold": 4, + "decision_owner": "lab", + "scheduler_owner": "orchestrator", + "not_a_deployment_decision": true, + "boundary_markers_valid": true + } + } + }, + { + "type": "runtime_history_seed_run_config_traceability", + "metric_name": "runtime_history_seed_run_config_runs", + "observed_value": 2, + "baseline_value": 2, + "threshold": 2, + "delta": 0, + "delta_pct": null, + "increase_factor": null, + "severity": "low", + "status": "passed", + "explanation": "Runtime history seed run_config markers are preserved for 2 of 2 expected Runtime history seed runs.", + "why_it_matters": "Runtime history seed run_config markers make EdgeEnv replay and comparability context traceable without making AIGuard or EdgeEnv the deployment decision owner.", + "suspected_causes": [], + "recommendation": "Preserve Runtime history seed run_config markers in the Lab report as traceability evidence.", + "raw_context": { + "history_seed_run_config": { + "marker_labels": [ + "baseline/candidate=shape=1x640x640, input_mode=dummy, input_preprocess=none, power_mode=unknown, jetson_clocks=unknown, warmup=1, runs=10" + ], + "history_seed_run_config_runs": 2, + "baseline_run_config_present": true, + "candidate_run_config_present": true, + "registry_owner": "edgeenv", + "decision_owner": "lab" + } + } + }, + { + "type": "runtime_thermal_instability", + "metric_name": "candidate_max_temperature_c", + "observed_value": 78.5, + "baseline_value": null, + "threshold": 70.0, + "delta": null, + "delta_pct": null, + "increase_factor": null, + "severity": "medium", + "status": "warning", + "explanation": "candidate_max_temperature_c observed value is 78.5. Threshold is 70. This runtime_thermal_instability evidence should be reviewed before deployment.", + "why_it_matters": "Thermal pressure or throttling in EdgeEnv telemetry can explain runtime latency drift and reduce sustained deployment reliability.", + "suspected_causes": [ + "thermal_pressure", + "thermal_throttling", + "power_mode_or_cooling_constraint" + ], + "recommendation": "Review EdgeEnv telemetry history, power mode, cooling, and sustained run conditions before treating runtime regression as stable.", + "raw_context": { + "edgeenv_regression": { + "baseline_run_id": "edgeenv-smoke-baseline", + "candidate_run_id": "edgeenv-smoke-candidate", + "comparable": true, + "mode": "same-condition", + "regression_detected": false, + "regression_type": "operation-context", + "severity": "medium", + "recommendation": "review_required", + "mean_delta_pct": null, + "p95_delta_pct": null, + "p99_delta_pct": null, + "fps_delta_pct": null, + "memory_peak_delta_pct": null, + "triggered_thresholds": [], + "runtime_telemetry_context_present": true, + "runtime_telemetry_source": "result_artifacts+runtime_telemetry_history", + "runtime_telemetry_history_schema_version": "edgeenv.runtime-telemetry-history.v1", + "history_orchestrator_feed_runs": 2.0, + "history_registered_runs": 3.0, + "history_telemetry_runs": 2.0, + "history_telemetry_seed_runs": 2.0, + "history_telemetry_seed_run_config_runs": 2.0, + "baseline_runtime_telemetry_history_seed_run_config": { + "batch": 1, + "height": 640, + "width": 640, + "warmup": 1, + "runs": 10, + "timeout_ms": null, + "input_mode": "dummy", + "input_preprocess": "none", + "power_mode": "unknown", + "jetson_clocks": "unknown" + }, + "candidate_runtime_telemetry_history_seed_run_config": { + "batch": 1, + "height": 640, + "width": 640, + "warmup": 1, + "runs": 10, + "timeout_ms": null, + "input_mode": "dummy", + "input_preprocess": "none", + "power_mode": "unknown", + "jetson_clocks": "unknown" + }, + "candidate_runtime_telemetry_history_seed_run_config_present": true, + "baseline_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", + "candidate_runtime_telemetry_history_seed_schema_version": "inferedge-runtime-telemetry-history-seed-v1", + "baseline_runtime_telemetry_history_seed_registry_owner": "edgeenv", + "candidate_runtime_telemetry_history_seed_registry_owner": "edgeenv", + "baseline_runtime_telemetry_history_seed_decision_owner": "lab", + "candidate_runtime_telemetry_history_seed_decision_owner": "lab", + "candidate_runtime_telemetry_history_seed_production_monitoring": false, + "candidate_runtime_telemetry_history_seed_missing_telemetry_is_failure": false, + "candidate_runtime_telemetry_history_seed_point_count": 1.0, + "history_missing_telemetry_runs": 1.0, + "telemetry_coverage_source": "history_telemetry_coverage", + "history_telemetry_coverage_missing_field_run_count": 1.0, + "history_telemetry_coverage_missing_field_runs": [ + { + "run_id": "edgeenv-smoke-candidate", + "missing_fields": [ + "queue_depth" + ], + "missing_field_count": 1, + "missing_telemetry_is_failure": false + } + ], + "history_telemetry_coverage_run_summaries_present": true, + "baseline_telemetry_present": true, + "candidate_telemetry_present": true, + "baseline_history_entry_present": true, + "candidate_history_entry_present": true, + "baseline_telemetry_coverage_ratio": 1.0, + "candidate_telemetry_coverage_ratio": 0.666667, + "baseline_telemetry_coverage_missing_fields": [], + "candidate_telemetry_coverage_missing_fields": [ + "queue_depth" + ], + "telemetry_coverage_missing_field_count": 1.0, + "baseline_missing_telemetry_is_failure": false, + "candidate_missing_telemetry_is_failure": false, + "baseline_execution_sequence_id": 1.0, + "candidate_execution_sequence_id": 2.0, + "execution_sequence_order_valid": true, + "baseline_orchestrator_context_present": false, + "candidate_orchestrator_context_present": true, "orchestrator_source_repository": "InferEdgeOrchestrator", "orchestrator_artifact_role": "orchestrator-supplemental-operation-context", "orchestrator_producer_contract": "inferedge-orchestrator-edgeenv-runtime-telemetry-feed-v1", @@ -2425,7 +3561,9 @@ "remote_dispatch_fallback_used", "queue_pressure_context", "worker_health_degradation_context", - "task_specific_queue_pressure_context" + "task_specific_queue_pressure_context", + "worker_health_constraint_context", + "worker_health_state_trend_context" ], "recommendations": [ "Inspect telemetry coverage missing fields, rerun telemetry history export if needed, and preserve runtime_telemetry artifacts for both baseline and candidate before relying on trend diagnosis.", @@ -2433,7 +3571,8 @@ "Review EdgeEnv telemetry history, power mode, cooling, and sustained run conditions before treating runtime regression as stable.", "Inspect Orchestrator queue policy, target FPS, drop/fallback behavior, and runtime telemetry before deployment.", "Review primary worker health, endpoint reachability, and fallback policy before treating the remote dispatch path as stable.", - "Review the Orchestrator operation risk summary in Lab alongside queue, worker health, and device-local producer evidence; Lab remains the final deployment decision owner." + "Review the Orchestrator operation risk summary in Lab alongside queue, worker health, and device-local producer evidence; Lab remains the final deployment decision owner.", + "Review worker_health_trend in Lab alongside scheduler fairness, queue pressure, and stale-drop evidence before treating the operation profile as stable." ], "thresholds": { "deadline_miss_rate_review": 0.05, @@ -2582,7 +3721,8 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "orchestrator_mapping_hint_copy_candidate_context_to": "runtime_telemetry_context.candidate", @@ -2597,14 +3737,16 @@ ], "orchestrator_mapping_hint_aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "orchestrator_downstream_guard_alignment": { "declared_by": "orchestrator", "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -2618,7 +3760,8 @@ "orchestrator_guard_alignment_producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "orchestrator_guard_alignment_operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "orchestrator_guard_alignment_orchestrator_is_final_decision_owner": false, "orchestrator_guard_alignment_lab_is_final_decision_owner": true, @@ -2805,7 +3948,224 @@ "vision_agent" ], "orchestrator_operation_risk_summary_device_local_event_count": 15.0, - "orchestrator_operation_risk_summary_producer_event_count": 7.0 + "orchestrator_operation_risk_summary_producer_event_count": 7.0, + "orchestrator_worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + }, + "orchestrator_worker_health_trend_present": true, + "orchestrator_worker_health_trend_schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "orchestrator_worker_health_trend_operation_context_role": "supplemental", + "orchestrator_worker_health_trend_scheduler_owner": "orchestrator", + "orchestrator_worker_health_trend_decision_owner": "lab", + "orchestrator_worker_health_trend_not_a_deployment_decision": true, + "orchestrator_worker_health_trend_health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "orchestrator_worker_health_trend_tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "orchestrator_worker_health_trend_task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "orchestrator_worker_health_trend_degraded_workers": [ + "vision_agent" + ], + "orchestrator_worker_health_trend_constrained_workers": [ + "voice_command_agent" + ], + "orchestrator_worker_health_trend_review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "orchestrator_operation_timeline_summary": { + "schema_version": "inferedge-orchestrator-operation-timeline-summary-v1", + "source": "queue_depth_timeline+latency_timeline+policy_decision_log+runtime_event_summary", + "sample_counts": { + "queue_depth": 2, + "latency": 2, + "policy_decision": 2, + "runtime_event": 15 + }, + "queue": { + "max_total_queue_depth": 7, + "average_total_queue_depth": 4.5, + "overload_backlog_threshold": 3, + "pressure_state": "overloaded", + "pressure_reason": "queue_backlog_threshold_exceeded", + "max_pressure_task": "vision_agent", + "max_queue_depth_by_task": { + "vision_agent": 7, + "voice_command_agent": 4 + } + }, + "latency": { + "sample_count": 2, + "max_latency_ms": 72.0, + "max_queue_wait_ms": 15.0, + "max_queue_wait_ms_by_task": { + "vision_agent": 15.0 + }, + "tasks_with_deadline_miss": [ + "vision_agent" + ] + }, + "policy": { + "decision_count": 2, + "decision_reasons": [ + "queue_backlog_threshold_exceeded" + ], + "first_decision": { + "decision_reason": "queue_backlog_threshold_exceeded" + }, + "latest_decision": { + "decision_reason": "queue_backlog_threshold_exceeded" + } + }, + "affected_tasks": { + "deadline_missed": [ + "vision_agent" + ], + "fallback": [ + "voice_command_agent" + ], + "scheduler_delay": [ + "vision_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "review_hints": [ + "review_queue_pressure", + "review_scheduler_delay", + "review_deadline_miss", + "review_fallback_use", + "review_worker_health_trend" + ], + "worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + } + }, + "orchestrator_operation_timeline_review_hints": [ + "review_queue_pressure", + "review_scheduler_delay", + "review_deadline_miss", + "review_fallback_use", + "review_worker_health_trend" + ], + "orchestrator_operation_timeline_affected_constrained_tasks": [ + "voice_command_agent" + ] }, "remote_dispatch": { "source_repository": "InferEdgeOrchestrator", diff --git a/examples/runtime_intelligence_chain/bundle_manifest.json b/examples/runtime_intelligence_chain/bundle_manifest.json index 52f13c1..2d21f0c 100644 --- a/examples/runtime_intelligence_chain/bundle_manifest.json +++ b/examples/runtime_intelligence_chain/bundle_manifest.json @@ -57,6 +57,7 @@ "AIGuard task event rollup evidence", "AIGuard operation timeline evidence", "AIGuard scheduler fairness evidence", + "AIGuard worker health trend evidence", "AIGuard policy pressure evidence", "AIGuard runtime operation anomalies", "AIGuard remote dispatch event summary", diff --git a/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json b/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json index df4679e..1a89b42 100644 --- a/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json +++ b/examples/runtime_intelligence_chain/edgeenv_lab_handoff_manifest.json @@ -68,7 +68,8 @@ "runtime_history_seed_run_config_traceability", "runtime_queue_overload", "runtime_thermal_instability", - "remote_execution_recovered_by_fallback" + "remote_execution_recovered_by_fallback", + "edgeenv_orchestrator_worker_health_trend" ], "optional_aiguard_evidence_types": [ "stale_frame_risk", @@ -119,7 +120,8 @@ "Remote fallback starter evidence", "lab=Remote fallback starter evidence; evidence=remote_execution_recovered_by_fallback", "AIGuard producer-lineage guard alignment", - "Lab remains the final deployment decision owner." + "Lab remains the final deployment decision owner.", + "AIGuard worker health trend evidence" ], "source_repositories": { "runtime_result": "InferEdge-Runtime", @@ -199,12 +201,17 @@ "orchestrator_operation_timeline_summary_present": true, "orchestrator_operation_timeline_summary_run_ids": [ "edgeenv-smoke-candidate" + ], + "orchestrator_worker_health_trend_present": true, + "orchestrator_worker_health_trend_run_ids": [ + "edgeenv-smoke-candidate" ] }, "notes": [ "This manifest is EdgeEnv producer-side handoff metadata.", "AIGuard guard_analysis is intentionally not produced by EdgeEnv.", "Lab remains the final deployment decision owner.", - "This is a local artifact contract, not production observability." + "This is a local artifact contract, not production observability.", + "EdgeEnv preserves Orchestrator worker_health_trend for AIGuard deterministic evidence and Lab report context." ] } diff --git a/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json b/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json index 0e5d3bf..2be65a0 100644 --- a/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json +++ b/examples/runtime_intelligence_chain/edgeenv_regression_with_orchestrator_context.json @@ -370,14 +370,64 @@ "degraded": [ "vision_agent" ], - "constrained": [] + "constrained": [ + "voice_command_agent" + ] }, "review_hints": [ "review_queue_pressure", "review_scheduler_delay", "review_deadline_miss", - "review_fallback_use" - ] + "review_fallback_use", + "review_worker_health_trend" + ], + "worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + } } }, "resource": { @@ -425,7 +475,8 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "downstream_guard_alignment": { @@ -433,7 +484,8 @@ "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -714,7 +766,8 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "downstream_guard_alignment": { @@ -722,7 +775,8 @@ "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", diff --git a/examples/runtime_intelligence_chain/runtime_telemetry_history.json b/examples/runtime_intelligence_chain/runtime_telemetry_history.json index ee4cafc..d8f028e 100644 --- a/examples/runtime_intelligence_chain/runtime_telemetry_history.json +++ b/examples/runtime_intelligence_chain/runtime_telemetry_history.json @@ -293,14 +293,64 @@ "degraded": [ "vision_agent" ], - "constrained": [] + "constrained": [ + "voice_command_agent" + ] }, "review_hints": [ "review_queue_pressure", "review_scheduler_delay", "review_deadline_miss", - "review_fallback_use" - ] + "review_fallback_use", + "review_worker_health_trend" + ], + "worker_health_trend": { + "schema_version": "inferedge-orchestrator-worker-health-trend-v1", + "operation_context_role": "supplemental", + "scheduler_owner": "orchestrator", + "decision_owner": "lab", + "not_a_deployment_decision": true, + "source": "worker_health_snapshot+runtime_event_summary", + "health_state_counts": { + "healthy": 1, + "degraded": 1, + "constrained": 1 + }, + "tasks_by_health_state": { + "healthy": [ + "safety_monitor_agent" + ], + "degraded": [ + "vision_agent" + ], + "constrained": [ + "voice_command_agent" + ] + }, + "task_health_context": { + "vision_agent": { + "worker_id": "vision_agent", + "health_state": "degraded", + "primary_health_reason": "worker_health_degraded" + }, + "voice_command_agent": { + "worker_id": "voice_command_agent", + "health_state": "constrained", + "primary_health_reason": "fallback_overuse" + } + }, + "degraded_workers": [ + "vision_agent" + ], + "constrained_workers": [ + "voice_command_agent" + ], + "review_hints": [ + "review_worker_health_degradation", + "review_worker_health_constraints" + ], + "interpretation": "Worker health trend is supplemental operation evidence; Lab remains the final deployment decision owner." + } } }, "resource": { @@ -343,7 +393,8 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "downstream_guard_alignment": { @@ -351,7 +402,8 @@ "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", @@ -438,7 +490,8 @@ ], "aiguard_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ] }, "downstream_guard_alignment": { @@ -446,7 +499,8 @@ "producer_lineage_evidence_type": "edgeenv_orchestrator_producer_lineage", "operation_evidence_candidates": [ "runtime_queue_overload", - "runtime_thermal_instability" + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend" ], "validated_by": [ "edgeenv runs telemetry inspect-history", diff --git a/inferedgelab/report/runtime_intelligence.py b/inferedgelab/report/runtime_intelligence.py index 1bf9135..b326b2a 100644 --- a/inferedgelab/report/runtime_intelligence.py +++ b/inferedgelab/report/runtime_intelligence.py @@ -34,6 +34,10 @@ "edgeenv_orchestrator_scheduler_fairness_summary" ) +ORCHESTRATOR_WORKER_HEALTH_TREND_EVIDENCE_TYPE = ( + "edgeenv_orchestrator_worker_health_trend" +) + ORCHESTRATOR_POLICY_PRESSURE_EVIDENCE_TYPE = ( "edgeenv_orchestrator_policy_pressure_summary" ) @@ -1568,6 +1572,16 @@ def _append_aiguard_runtime_operation_rows( ) ) + worker_health_trend_label = _aiguard_worker_health_trend_label(evidence_items) + if worker_health_trend_label: + rows.append( + ( + "AIGuard worker health trend evidence", + worker_health_trend_label, + "AIGuard preserves degraded/constrained worker trend context as deterministic review evidence; Lab still owns the deployment decision.", + ) + ) + policy_pressure_label = _aiguard_policy_pressure_label(evidence_items) if policy_pressure_label: rows.append( @@ -2051,6 +2065,51 @@ def _aiguard_scheduler_fairness_label( return ", ".join(parts) +def _aiguard_worker_health_trend_label( + evidence_items: list[dict[str, Any]], +) -> str: + evidence = _find_evidence_item( + evidence_items, + ORCHESTRATOR_WORKER_HEALTH_TREND_EVIDENCE_TYPE, + ) + if evidence is None: + return "" + + parts: list[str] = [] + status = evidence.get("status") + if status is not None: + parts.append(f"status={status}") + observed = evidence.get("observed_value") + if observed is not None: + parts.append(f"markers={_format_compact_value(observed)}") + + raw_context = evidence.get("raw_context") + if isinstance(raw_context, dict): + context = raw_context.get("worker_health_trend") + if isinstance(context, dict): + degraded_workers = _string_list(context.get("degraded_workers")) + if degraded_workers: + parts.append("degraded=" + ",".join(degraded_workers)) + constrained_workers = _string_list(context.get("constrained_workers")) + if constrained_workers: + parts.append("constrained=" + ",".join(constrained_workers)) + health_state_counts = context.get("health_state_counts") + if isinstance(health_state_counts, dict): + state_labels = [ + f"{state}:{_format_compact_value(count)}" + for state, count in health_state_counts.items() + if isinstance(state, str) + and state in {"healthy", "degraded", "constrained"} + and count is not None + ] + if state_labels: + parts.append("health_states=" + ",".join(state_labels)) + boundary_valid = context.get("boundary_markers_valid") + if boundary_valid is not None: + parts.append(f"boundary_valid={boundary_valid}") + return ", ".join(parts) + + def _aiguard_policy_pressure_label( evidence_items: list[dict[str, Any]], ) -> str: diff --git a/scripts/check_runtime_intelligence_artifact_bundle.py b/scripts/check_runtime_intelligence_artifact_bundle.py index 9bd96d8..ad5863b 100644 --- a/scripts/check_runtime_intelligence_artifact_bundle.py +++ b/scripts/check_runtime_intelligence_artifact_bundle.py @@ -144,7 +144,7 @@ "aiguard_task_event_rollup_type": "edgeenv_orchestrator_task_event_rollup", "aiguard_operation_timeline_evidence": ( "| AIGuard operation timeline evidence | " - "status=warning, markers=6" + "status=warning, markers=7" ), "aiguard_operation_timeline_type": ( "edgeenv_orchestrator_operation_timeline_summary" @@ -156,6 +156,13 @@ "aiguard_scheduler_fairness_type": ( "edgeenv_orchestrator_scheduler_fairness_summary" ), + "aiguard_worker_health_trend_evidence": ( + "| AIGuard worker health trend evidence | " + "status=warning, markers=5" + ), + "aiguard_worker_health_trend_type": ( + "edgeenv_orchestrator_worker_health_trend" + ), "aiguard_policy_pressure_evidence": ( "| AIGuard policy pressure evidence | " "status=warning, markers=5" @@ -292,6 +299,12 @@ "aiguard_scheduler_fairness_type": ( "edgeenv_orchestrator_scheduler_fairness_summary" ), + "aiguard_worker_health_trend_evidence": ( + "AIGuard worker health trend evidence" + ), + "aiguard_worker_health_trend_type": ( + "edgeenv_orchestrator_worker_health_trend" + ), "aiguard_policy_pressure_evidence": "AIGuard policy pressure evidence", "aiguard_policy_pressure_type": ( "edgeenv_orchestrator_policy_pressure_summary" diff --git a/scripts/check_runtime_intelligence_bundle_manifest.py b/scripts/check_runtime_intelligence_bundle_manifest.py index d9cd228..0fd8da4 100644 --- a/scripts/check_runtime_intelligence_bundle_manifest.py +++ b/scripts/check_runtime_intelligence_bundle_manifest.py @@ -73,6 +73,7 @@ REQUIRED_ORCHESTRATOR_AIGUARD_EVIDENCE_CANDIDATES = { "runtime_queue_overload", "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend", } REQUIRED_ORCHESTRATOR_GUARD_ALIGNMENT = { "declared_by": "orchestrator", @@ -101,6 +102,7 @@ "edgeenv_orchestrator_task_event_rollup", "edgeenv_orchestrator_operation_timeline_summary", "edgeenv_orchestrator_scheduler_fairness_summary", + "edgeenv_orchestrator_worker_health_trend", "edgeenv_orchestrator_policy_pressure_summary", "runtime_history_seed_run_config_traceability", "runtime_queue_overload", @@ -178,6 +180,7 @@ "AIGuard task event rollup evidence", "AIGuard operation timeline evidence", "AIGuard scheduler fairness evidence", + "AIGuard worker health trend evidence", "AIGuard policy pressure evidence", "AIGuard runtime operation anomalies", "AIGuard remote dispatch event summary", @@ -197,7 +200,7 @@ "ownership: regression_owner=edgeenv, deployment_decision_owner=lab", "orchestrator_mapping_hint: coverage_summary_owner=edgeenv", "orchestrator_mapping_hint: operation_context_role=supplemental", - "orchestrator_mapping_hint: aiguard_evidence_candidates=runtime_queue_overload,runtime_thermal_instability", + "orchestrator_mapping_hint: aiguard_evidence_candidates=runtime_queue_overload,runtime_thermal_instability,edgeenv_orchestrator_worker_health_trend", "orchestrator_downstream_guard_alignment: producer_lineage_evidence_type=edgeenv_orchestrator_producer_lineage", "orchestrator_device_local_producer_lineage: candidate_context.producer validated", "orchestrator_producer_lineage_shape: per-task source/stage/count mappings validated", @@ -207,12 +210,14 @@ "aiguard_evidence: edgeenv_orchestrator_task_event_rollup validated", "aiguard_evidence: edgeenv_orchestrator_operation_timeline_summary validated", "aiguard_evidence: edgeenv_orchestrator_scheduler_fairness_summary validated", + "aiguard_evidence: edgeenv_orchestrator_worker_health_trend validated", "aiguard_evidence: edgeenv_orchestrator_policy_pressure_summary validated", "aiguard_evidence: runtime_history_seed_run_config_traceability validated", "aiguard_evidence: remote_execution_recovered_by_fallback validated", "aiguard_raw_context: producer_lineage_shape preserved", "aiguard_raw_context: task_event_rollup preserved", "aiguard_raw_context: scheduler_fairness_summary preserved", + "aiguard_raw_context: worker_health_trend preserved", "aiguard_raw_context: policy_pressure_summary preserved", "aiguard_raw_context: history_seed_run_config_traceability preserved", "aiguard_raw_context: remote_runtime_event_summary preserved", @@ -243,6 +248,7 @@ "edgeenv_handoff: orchestrator_operation_risk_rollup validated", "edgeenv_handoff: orchestrator_task_event_rollup validated", "edgeenv_handoff: orchestrator_operation_timeline_summary validated", + "edgeenv_handoff: orchestrator_worker_health_trend validated", "edgeenv_handoff: missing_telemetry_orchestrator_context validated", "edgeenv_handoff: external AIGuard evidence requirements declared", ) @@ -685,6 +691,9 @@ def _validate_edgeenv_handoff_report_summary( operation_timeline_summary_run_ids = ( _orchestrator_operation_timeline_summary_run_ids(context) ) + worker_health_trend_run_ids = _orchestrator_worker_health_trend_run_ids( + context + ) fixture_matrix_context = regression_report.get("fixture_matrix_context") fixture_matrix_summary = _fixture_matrix_summary(fixture_matrix_context) @@ -767,6 +776,22 @@ def _validate_edgeenv_handoff_report_summary( "orchestrator_operation_timeline_summary_run_ids must match preserved " "Orchestrator operation timeline summary run IDs", ) + _record( + summary.get("orchestrator_worker_health_trend_present") + is bool(worker_health_trend_run_ids), + errors, + "EdgeEnv handoff edgeenv_report_summary." + "orchestrator_worker_health_trend_present must match preserved " + "Orchestrator worker health trend context", + ) + _record( + summary.get("orchestrator_worker_health_trend_run_ids") + == worker_health_trend_run_ids, + errors, + "EdgeEnv handoff edgeenv_report_summary." + "orchestrator_worker_health_trend_run_ids must match preserved " + "Orchestrator worker health trend run IDs", + ) _record( summary.get("fixture_matrix_context_present") is bool(fixture_matrix_summary), @@ -859,6 +884,14 @@ def _orchestrator_operation_timeline_summary_run_ids(context: Any) -> list[str]: ] +def _orchestrator_worker_health_trend_run_ids(context: Any) -> list[str]: + return [ + run_id + for run_id, operation_context in _runtime_operation_contexts(context) + if _has_orchestrator_worker_health_trend(operation_context) + ] + + def _runtime_operation_contexts(context: Any) -> list[tuple[str, dict[str, Any]]]: if not isinstance(context, dict): return [] @@ -949,6 +982,18 @@ def _has_orchestrator_operation_timeline_summary( return isinstance(timeline, dict) and bool(timeline) +def _has_orchestrator_worker_health_trend( + operation_context: dict[str, Any], +) -> bool: + operation = _orchestrator_candidate_operation(operation_context) + trend = operation.get("worker_health_trend") + if not isinstance(trend, dict): + timeline = operation.get("operation_timeline_summary") + if isinstance(timeline, dict): + trend = timeline.get("worker_health_trend") + return isinstance(trend, dict) and bool(trend) + + def _orchestrator_candidate_operation( operation_context: dict[str, Any], ) -> dict[str, Any]: @@ -1897,6 +1942,8 @@ def _validate_guard_analysis(guard_analysis: dict[str, Any], errors: list[str]) _validate_operation_timeline_evidence(item, index, errors) if item.get("type") == "edgeenv_orchestrator_scheduler_fairness_summary": _validate_scheduler_fairness_evidence(item, index, errors) + if item.get("type") == "edgeenv_orchestrator_worker_health_trend": + _validate_worker_health_trend_evidence(item, index, errors) if item.get("type") == "edgeenv_orchestrator_policy_pressure_summary": _validate_policy_pressure_evidence(item, index, errors) if item.get("type") == "runtime_history_seed_run_config_traceability": @@ -2335,9 +2382,9 @@ def _validate_operation_timeline_evidence( f"AIGuard evidence[{index}] operation timeline status must be warning", ) _record( - item.get("observed_value") == 6, + item.get("observed_value") == 7, errors, - f"AIGuard evidence[{index}] operation timeline observed_value must be 6", + f"AIGuard evidence[{index}] operation timeline observed_value must be 7", ) raw_context = item.get("raw_context") or {} timeline = raw_context.get("operation_timeline_summary") @@ -2391,6 +2438,7 @@ def _validate_operation_timeline_evidence( "review_scheduler_delay", "review_deadline_miss", "review_fallback_use", + "review_worker_health_trend", } - set(review_hints) ) @@ -2497,6 +2545,72 @@ def _validate_scheduler_fairness_evidence( ) +def _validate_worker_health_trend_evidence( + item: dict[str, Any], + index: int, + errors: list[str], +) -> None: + _record( + item.get("status") == "warning", + errors, + f"AIGuard evidence[{index}] worker health trend status must be warning", + ) + _record( + item.get("observed_value") == 5, + errors, + f"AIGuard evidence[{index}] worker health trend observed_value must be 5", + ) + raw_context = item.get("raw_context") or {} + trend = raw_context.get("worker_health_trend") + _record( + isinstance(trend, dict), + errors, + f"AIGuard evidence[{index}] raw_context.worker_health_trend must be an object", + ) + if not isinstance(trend, dict): + return + + _record( + trend.get("boundary_markers_valid") is True, + errors, + f"AIGuard evidence[{index}] worker_health_trend.boundary_markers_valid must be true", + ) + _record( + trend.get("degraded_workers") == ["vision_agent"], + errors, + f"AIGuard evidence[{index}] worker_health_trend.degraded_workers must preserve vision_agent", + ) + _record( + trend.get("constrained_workers") == ["voice_command_agent"], + errors, + f"AIGuard evidence[{index}] worker_health_trend.constrained_workers must preserve voice_command_agent", + ) + state_counts = trend.get("health_state_counts") + _record( + isinstance(state_counts, dict) + and state_counts.get("healthy") == 1 + and state_counts.get("degraded") == 1 + and state_counts.get("constrained") == 1, + errors, + f"AIGuard evidence[{index}] worker_health_trend.health_state_counts must preserve healthy/degraded/constrained counts", + ) + _record( + trend.get("decision_owner") == "lab", + errors, + f"AIGuard evidence[{index}] worker_health_trend.decision_owner must be lab", + ) + _record( + trend.get("scheduler_owner") == "orchestrator", + errors, + f"AIGuard evidence[{index}] worker_health_trend.scheduler_owner must be orchestrator", + ) + _record( + trend.get("not_a_deployment_decision") is True, + errors, + f"AIGuard evidence[{index}] worker_health_trend.not_a_deployment_decision must be true", + ) + + def _validate_policy_pressure_evidence( item: dict[str, Any], index: int, diff --git a/tests/test_runtime_intelligence_bundle_manifest.py b/tests/test_runtime_intelligence_bundle_manifest.py index 1b3f617..6c6ed99 100644 --- a/tests/test_runtime_intelligence_bundle_manifest.py +++ b/tests/test_runtime_intelligence_bundle_manifest.py @@ -1103,7 +1103,10 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_mapping_hint( mapping_hint = operation_context["edgeenv_mapping_hint"] mapping_hint["coverage_summary_owner"] = "orchestrator" mapping_hint.pop("candidate_context_required_fields") - mapping_hint["aiguard_evidence_candidates"] = ["runtime_queue_overload"] + mapping_hint["aiguard_evidence_candidates"] = [ + "runtime_queue_overload", + "edgeenv_orchestrator_worker_health_trend", + ] operation_context["candidate_context"].pop("telemetry_source") edgeenv_copy = tmp_path / "edgeenv_regression.json" @@ -1446,7 +1449,7 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_guard_mapping_h ] = "aiguard" edgeenv_context["orchestrator_edgeenv_mapping_hint"][ "aiguard_evidence_candidates" - ] = ["runtime_queue_overload"] + ] = ["runtime_queue_overload", "edgeenv_orchestrator_worker_health_trend"] edgeenv_context.pop("orchestrator_mapping_hint_candidate_context_required_fields") edgeenv_context.pop("orchestrator_mapping_hint_aiguard_evidence_candidates") edgeenv_context["orchestrator_candidate_context_telemetry_source"] = "unknown" @@ -1648,10 +1651,11 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_guard_operation assert result == 2 summary = summary_path.read_text(encoding="utf-8") - assert "operation timeline observed_value must be 6" in summary + assert "operation timeline observed_value must be 7" in summary assert ( "operation_timeline_summary.review_hints is missing " - "['review_deadline_miss', 'review_fallback_use', 'review_scheduler_delay']" + "['review_deadline_miss', 'review_fallback_use', " + "'review_scheduler_delay', 'review_worker_health_trend']" ) in summary assert "operation_timeline_summary.decision_owner must be lab" in summary assert ( @@ -1685,7 +1689,7 @@ def test_runtime_intelligence_bundle_manifest_gate_fails_for_bad_guard_missing_c ] = "aiguard" edgeenv_context[ "history_missing_orchestrator_mapping_hint_aiguard_evidence_candidates" - ] = ["runtime_queue_overload"] + ] = ["runtime_queue_overload", "edgeenv_orchestrator_worker_health_trend"] guard_copy = tmp_path / "aiguard_guard_analysis.json" guard_copy.write_text(json.dumps(guard_analysis), encoding="utf-8") diff --git a/tests/test_runtime_intelligence_evidence_chain_smoke.py b/tests/test_runtime_intelligence_evidence_chain_smoke.py index 5c3b41c..e0cceca 100644 --- a/tests/test_runtime_intelligence_evidence_chain_smoke.py +++ b/tests/test_runtime_intelligence_evidence_chain_smoke.py @@ -103,6 +103,7 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): assert "edgeenv_orchestrator_operation_risk_rollup" in evidence_types assert "edgeenv_orchestrator_task_event_rollup" in evidence_types assert "edgeenv_orchestrator_operation_timeline_summary" in evidence_types + assert "edgeenv_orchestrator_worker_health_trend" in evidence_types assert "runtime_history_seed_run_config_traceability" in evidence_types assert "remote_execution_recovered_by_fallback" in evidence_types coverage_evidence = next( @@ -166,7 +167,11 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): ] == "edgeenv_orchestrator_producer_lineage" assert guard_edgeenv_context[ "orchestrator_guard_alignment_operation_evidence_candidates" - ] == ["runtime_queue_overload", "runtime_thermal_instability"] + ] == [ + "runtime_queue_overload", + "runtime_thermal_instability", + "edgeenv_orchestrator_worker_health_trend", + ] assert ( guard_edgeenv_context[ "orchestrator_guard_alignment_lab_is_final_decision_owner" @@ -316,7 +321,7 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): if item["type"] == "edgeenv_orchestrator_operation_timeline_summary" ) assert operation_timeline_evidence["status"] == "warning" - assert operation_timeline_evidence["observed_value"] == 6 + assert operation_timeline_evidence["observed_value"] == 7 operation_timeline_context = operation_timeline_evidence["raw_context"][ "operation_timeline_summary" ] @@ -329,6 +334,29 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): assert operation_timeline_context["policy_decision_reasons"] == [ "queue_backlog_threshold_exceeded" ] + worker_health_trend_evidence = next( + item + for item in bundle["guard_analysis"]["evidence"] + if item["type"] == "edgeenv_orchestrator_worker_health_trend" + ) + assert worker_health_trend_evidence["status"] == "warning" + assert worker_health_trend_evidence["observed_value"] == 5 + worker_health_trend_context = worker_health_trend_evidence["raw_context"][ + "worker_health_trend" + ] + assert worker_health_trend_context["boundary_markers_valid"] is True + assert worker_health_trend_context["scheduler_owner"] == "orchestrator" + assert worker_health_trend_context["decision_owner"] == "lab" + assert worker_health_trend_context["not_a_deployment_decision"] is True + assert worker_health_trend_context["degraded_workers"] == ["vision_agent"] + assert worker_health_trend_context["constrained_workers"] == [ + "voice_command_agent" + ] + assert worker_health_trend_context["health_state_counts"] == { + "healthy": 1, + "degraded": 1, + "constrained": 1, + } assert operation_risk_context["device_local_event_count"] == 15.0 run_config_traceability_evidence = next( item @@ -454,7 +482,8 @@ def test_runtime_intelligence_chain_smoke_ingests_precomputed_guard_artifact(): assert ( "| AIGuard producer-lineage guard alignment | " "evidence=edgeenv_orchestrator_producer_lineage, " - "candidates=runtime_queue_overload,runtime_thermal_instability, " + "candidates=runtime_queue_overload,runtime_thermal_instability," + "edgeenv_orchestrator_worker_health_trend, " "lab_final_owner=true |" ) in bundle["markdown"] assert "| AIGuard history seed handoff | seeds=2.0" in bundle["markdown"] @@ -585,6 +614,11 @@ def test_compare_cmd_runtime_intelligence_chain_writes_markdown_and_html( assert "max_wait_ms=15" in markdown assert "policy_decisions=2" in markdown assert "status=warning, markers=4" in markdown + assert "AIGuard worker health trend evidence" in markdown + assert "edgeenv_orchestrator_worker_health_trend" in markdown + assert "degraded=vision_agent" in markdown + assert "constrained=voice_command_agent" in markdown + assert "health_states=healthy:1,degraded:1,constrained:1" in markdown assert "health=worker_health_degraded" in markdown assert "Orchestrator context attached runs" in markdown assert "Orchestrator operation risk summary" in markdown @@ -711,6 +745,11 @@ def test_compare_cmd_runtime_intelligence_chain_writes_markdown_and_html( assert "tasks=vision_agent,voice_command_agent" in html assert "max_wait_ms=15" in html assert "policy_decisions=2" in html + assert "AIGuard worker health trend evidence" in html + assert "edgeenv_orchestrator_worker_health_trend" in html + assert "degraded=vision_agent" in html + assert "constrained=voice_command_agent" in html + assert "health_states=healthy:1,degraded:1,constrained:1" in html assert "health=worker_health_degraded" in html assert "AIGuard remote dispatch event summary" in html assert "events=3, final=succeeded, fallback_recovered=True" in html