From 875e1a563221cf0ec421c783f819a078faf45808 Mon Sep 17 00:00:00 2001 From: nickgee31 Date: Wed, 29 Jul 2026 17:08:05 +0100 Subject: [PATCH] Auto-restart unhealthy components Add automatic restart support for unhealthy components. Mark LoadML component with "auto_restart": True. Components now track restart_tasks and provide auto_restart() to schedule a single automatic restart via base.create_task (avoids double-scheduling). PredBat triggers auto_restart for non-calculating failed components. Update tests (FakeComponents) to record and assert auto-restart behavior, including ensuring calculating components are not restarted. Improves resilience by rebooting unhealthy components while avoiding interference during active calculations. --- apps/predbat/components.py | 14 +++++++++++++ apps/predbat/predbat.py | 2 ++ .../tests/test_component_health_status.py | 21 +++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/apps/predbat/components.py b/apps/predbat/components.py index 7eb6405ab..11175d38a 100644 --- a/apps/predbat/components.py +++ b/apps/predbat/components.py @@ -487,6 +487,7 @@ "class": LoadMLComponent, "name": "ML Load Forecaster", "event_filter": "predbat_load_ml_", + "auto_restart": True, "args": { "load_ml_enable": {"required_true": True, "config": "load_ml_enable", "default": False}, "load_ml_source": {"required": False, "config": "load_ml_source", "default": False}, @@ -528,6 +529,7 @@ class Components: def __init__(self, base): self.components = {} self.component_tasks = {} + self.restart_tasks = {} self.base = base self.log = base.log @@ -624,6 +626,18 @@ async def restart(self, only): for phase in (0, 1, 2): self.start(only=only, phase=phase) + def auto_restart(self, only): + """Schedule one automatic restart for an unhealthy component.""" + if not self.can_restart(only) or not self.is_active(only): + return False + restart_task = self.restart_tasks.get(only) + if restart_task and restart_task.is_alive(): + self.log(f"Info: Automatic restart already active for {only}") + return False + self.log(f"Warn: Automatically restarting unhealthy {only} component") + self.restart_tasks[only] = self.base.create_task(self.restart(only), name=f"{only}_auto_restart_task") + return True + """ Pass through events to the appropriate component """ diff --git a/apps/predbat/predbat.py b/apps/predbat/predbat.py index 0b8ac3a1c..ce916d945 100644 --- a/apps/predbat/predbat.py +++ b/apps/predbat/predbat.py @@ -766,6 +766,8 @@ def record_final_run_status(self, status, status_extra): component = self.components.get_component(component_name) if not component.is_calculating(): failed_components.append(COMPONENT_LIST.get(component_name, {}).get("name", component_name)) + if COMPONENT_LIST.get(component_name, {}).get("auto_restart", False): + self.components.auto_restart(component_name) elif is_active: component_status[component_name] = "running" else: diff --git a/apps/predbat/tests/test_component_health_status.py b/apps/predbat/tests/test_component_health_status.py index 2c59d5ae3..46ebbd898 100644 --- a/apps/predbat/tests/test_component_health_status.py +++ b/apps/predbat/tests/test_component_health_status.py @@ -27,6 +27,7 @@ class FakeComponents: def __init__(self, alive_map, calculating_map=None): self.alive_map = alive_map self.calculating_map = calculating_map or {} + self.auto_restart_calls = [] def get_all(self): return list(self.alive_map.keys()) @@ -47,6 +48,10 @@ def get_component(self, name): """Return a fake component with the configured calculation state.""" return FakeComponent(self.calculating_map.get(name, False)) + def auto_restart(self, name): + """Record an automatic restart request.""" + self.auto_restart_calls.append(name) + def test_component_health_status(my_predbat): """ @@ -134,6 +139,22 @@ def test_component_health_status(my_predbat): else: print("OK: LoadML failure outside calculation still fails the run status") + if my_predbat.components.auto_restart_calls != ["load_ml"]: + print("ERROR: Non-calculating LoadML failure did not request an automatic restart: {}".format(my_predbat.components.auto_restart_calls)) + failed = 1 + else: + print("OK: Non-calculating LoadML failure requests an automatic restart") + + # --- A calculating LoadML component must not be restarted --- + my_predbat.components = FakeComponents({"load_ml": False}, calculating_map={"load_ml": True}) + recorded_statuses.clear() + my_predbat.record_final_run_status("Export", "") + if my_predbat.components.auto_restart_calls: + print("ERROR: Calculating LoadML component requested an automatic restart") + failed = 1 + else: + print("OK: Calculating LoadML component is not automatically restarted") + # --- Pre-existing error takes precedence, and is not overwritten by the component check --- my_predbat.had_errors = True my_predbat.components = FakeComponents({"octopus": False})