Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions apps/predbat/components.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,7 @@
"class": LoadMLComponent,
"name": "ML Load Forecaster",
"event_filter": "predbat_load_ml_",
"auto_restart": True,
"args": {
"load_ml_enable": {"required_true": True, "config": "load_ml_enable", "default": False},
"load_ml_source": {"required": False, "config": "load_ml_source", "default": False},
Expand Down Expand Up @@ -528,6 +529,7 @@ class Components:
def __init__(self, base):
self.components = {}
self.component_tasks = {}
self.restart_tasks = {}
self.base = base
self.log = base.log

Expand Down Expand Up @@ -624,6 +626,18 @@ async def restart(self, only):
for phase in (0, 1, 2):
self.start(only=only, phase=phase)

def auto_restart(self, only):
"""Schedule one automatic restart for an unhealthy component."""
if not self.can_restart(only) or not self.is_active(only):
return False
restart_task = self.restart_tasks.get(only)
if restart_task and restart_task.is_alive():
self.log(f"Info: Automatic restart already active for {only}")
return False
self.log(f"Warn: Automatically restarting unhealthy {only} component")
self.restart_tasks[only] = self.base.create_task(self.restart(only), name=f"{only}_auto_restart_task")
return True

"""
Pass through events to the appropriate component
"""
Expand Down
2 changes: 2 additions & 0 deletions apps/predbat/predbat.py
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,8 @@ def record_final_run_status(self, status, status_extra):
component = self.components.get_component(component_name)
if not component.is_calculating():
failed_components.append(COMPONENT_LIST.get(component_name, {}).get("name", component_name))
if COMPONENT_LIST.get(component_name, {}).get("auto_restart", False):
self.components.auto_restart(component_name)
elif is_active:
component_status[component_name] = "running"
else:
Expand Down
21 changes: 21 additions & 0 deletions apps/predbat/tests/test_component_health_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class FakeComponents:
def __init__(self, alive_map, calculating_map=None):
self.alive_map = alive_map
self.calculating_map = calculating_map or {}
self.auto_restart_calls = []

def get_all(self):
return list(self.alive_map.keys())
Expand All @@ -47,6 +48,10 @@ def get_component(self, name):
"""Return a fake component with the configured calculation state."""
return FakeComponent(self.calculating_map.get(name, False))

def auto_restart(self, name):
"""Record an automatic restart request."""
self.auto_restart_calls.append(name)


def test_component_health_status(my_predbat):
"""
Expand Down Expand Up @@ -134,6 +139,22 @@ def test_component_health_status(my_predbat):
else:
print("OK: LoadML failure outside calculation still fails the run status")

if my_predbat.components.auto_restart_calls != ["load_ml"]:
print("ERROR: Non-calculating LoadML failure did not request an automatic restart: {}".format(my_predbat.components.auto_restart_calls))
failed = 1
else:
print("OK: Non-calculating LoadML failure requests an automatic restart")

# --- A calculating LoadML component must not be restarted ---
my_predbat.components = FakeComponents({"load_ml": False}, calculating_map={"load_ml": True})
recorded_statuses.clear()
my_predbat.record_final_run_status("Export", "")
if my_predbat.components.auto_restart_calls:
print("ERROR: Calculating LoadML component requested an automatic restart")
failed = 1
else:
print("OK: Calculating LoadML component is not automatically restarted")

# --- Pre-existing error takes precedence, and is not overwritten by the component check ---
my_predbat.had_errors = True
my_predbat.components = FakeComponents({"octopus": False})
Expand Down
Loading