diff --git a/apps/predbat/components.py b/apps/predbat/components.py index 6d043840a..ee389e50e 100644 --- a/apps/predbat/components.py +++ b/apps/predbat/components.py @@ -107,6 +107,7 @@ "forecast_solar": {"required": False, "config": "forecast_solar", "default": False}, "forecast_solar_max_age": {"required": False, "config": "forecast_solar_max_age", "default": 8}, "forecast_solar_open_meteo_backup": {"required": False, "config": "forecast_solar_open_meteo_backup", "default": False}, + "forecast_solar_open_meteo_first": {"required": False, "config": "forecast_solar_open_meteo_first", "default": False}, "pv_forecast_today": {"required": False, "config": "pv_forecast_today"}, "pv_forecast_tomorrow": {"required": False, "config": "pv_forecast_tomorrow"}, "pv_forecast_d3": {"required": False, "config": "pv_forecast_d3"}, diff --git a/apps/predbat/config.py b/apps/predbat/config.py index f2acafc86..74fac9e94 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -2365,6 +2365,7 @@ "forecast_solar": {"type": "dict_list"}, "forecast_solar_max_age": {"type": "float"}, "forecast_solar_open_meteo_backup": {"type": "boolean"}, + "forecast_solar_open_meteo_first": {"type": "boolean"}, "open_meteo_forecast": {"type": "dict_list"}, "open_meteo_forecast_max_age": {"type": "float"}, "enable_coarse_fine_levels": {"type": "boolean"}, diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index cae9233f0..e0c4bf22a 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -55,6 +55,7 @@ def initialize( forecast_solar, forecast_solar_max_age, forecast_solar_open_meteo_backup, + forecast_solar_open_meteo_first, pv_forecast_today, pv_forecast_tomorrow, pv_forecast_d3, @@ -71,6 +72,7 @@ def initialize( self.forecast_solar = forecast_solar self.forecast_solar_max_age = forecast_solar_max_age self.forecast_solar_open_meteo_backup = forecast_solar_open_meteo_backup + self.forecast_solar_open_meteo_first = forecast_solar_open_meteo_first self.pv_forecast_today = pv_forecast_today self.pv_forecast_tomorrow = pv_forecast_tomorrow self.pv_forecast_d3 = pv_forecast_d3 @@ -1201,6 +1203,20 @@ def pack_and_store_forecast(self, pv_forecast_minute, pv_forecast_minute10): app="solar", ) + async def log_source_change(self, source): + """Warn when the configured solar forecast source changes so the PV calibration settling period is visible.""" + if not source: + return + if not self.storage: + return + stored = await self.storage.load("solcast", "active_forecast_source") + previous = stored.get("source") if isinstance(stored, dict) else None + if previous == source: + return + if previous: + self.log("Warn: SolarAPI: Configured solar forecast source changed from {} to {}, PV calibration will settle over the next 7 days".format(previous, source)) + await self.storage.save("solcast", "active_forecast_source", {"source": source}, format="json", expiry=None) + async def fetch_pv_forecast(self): """ Fetch the PV Forecast data from Solcast @@ -1212,14 +1228,26 @@ async def fetch_pv_forecast(self): pv_forecast_total_data = 0 pv_forecast_total_sensor = 0 create_pv10 = False + configured_source = None max_kwh = 9999 using_ha_data = False - if self.forecast_solar: + if self.forecast_solar and self.forecast_solar_open_meteo_first: + self.log("SolarAPI: Obtaining solar forecast from Open-Meteo API (primary, Forecast Solar fallback)") + primary_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar + pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=primary_configs) + divide_by = 30.0 + create_pv10 = True + configured_source = "open_meteo" + if not pv_forecast_data: + self.log("Warn: SolarAPI: Open-Meteo returned no data, falling back to Forecast Solar") + pv_forecast_data, max_kwh = await self.download_forecast_solar_data() + elif self.forecast_solar: self.log("SolarAPI: Obtaining solar forecast from Forecast Solar API") pv_forecast_data, max_kwh = await self.download_forecast_solar_data() divide_by = 30.0 create_pv10 = True + configured_source = "forecast_solar" if not pv_forecast_data and self.forecast_solar_open_meteo_backup: self.log("SolarAPI: Forecast Solar returned no data, falling back to Open-Meteo backup") backup_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar @@ -1229,13 +1257,16 @@ async def fetch_pv_forecast(self): pv_forecast_data, max_kwh = await self.download_open_meteo_data() divide_by = 30.0 create_pv10 = True + configured_source = "open_meteo" elif self.solcast_host and self.solcast_api_key: self.log("SolarAPI: Obtaining solar forecast from Solcast API") pv_forecast_data = await self.download_solcast_data() divide_by = 30.0 + configured_source = "solcast" else: self.log("SolarAPI: Using Solcast integration from inside HA for solar forecast") using_ha_data = True + configured_source = "ha_sensors" # Fetch data from each sensor for argname in ["pv_forecast_today", "pv_forecast_tomorrow", "pv_forecast_d3", "pv_forecast_d4"]: @@ -1265,6 +1296,8 @@ async def fetch_pv_forecast(self): self.log("SolarAPI: PV Forecast today adds up to {} kWh, and total sensors add up to {} kWh, factor is {}".format(pv_forecast_total_data, pv_forecast_total_sensor, factor)) if pv_forecast_data: + await self.log_source_change(configured_source) + # Detect the actual period of the forecast data (e.g. 15 or 30 minutes) # by examining the time difference between consecutive entries. # This ensures 15-minute resolution data is handled correctly. diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index 573dd8b04..e07dd5cc5 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -169,6 +169,7 @@ def __init__(self): forecast_solar=None, forecast_solar_max_age=4, forecast_solar_open_meteo_backup=False, + forecast_solar_open_meteo_first=False, pv_forecast_today=None, pv_forecast_tomorrow=None, pv_forecast_d3=None, @@ -1827,6 +1828,407 @@ def create_mock_session(*args, **kwargs): return failed +def test_fetch_pv_forecast_open_meteo_first_used_when_set(my_predbat): + """ + When forecast_solar_open_meteo_first is True, Open-Meteo is used as the primary + source and forecast.solar is not called at all. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_used_when_set") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + # Both sources would succeed - Open-Meteo must win and forecast.solar must not be called + test_api.set_mock_response( + "api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"], + "global_tilted_irradiance": [500.0, 600.0, 550.0], + "temperature_2m": [25.0, 25.0, 25.0], + "wind_speed_10m": [1.0, 1.0, 1.0], + } + }, + ) + test_api.set_mock_response( + "ensemble-api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"], + "global_tilted_irradiance_member01": [400.0, 480.0, 440.0], + } + }, + ) + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + open_meteo_calls = [r for r in test_api.request_log if "open-meteo.com" in r["url"]] + if len(open_meteo_calls) == 0: + print("ERROR: Expected Open-Meteo API call when open_meteo_first is set, got none") + failed = True + + forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]] + if len(forecast_calls) != 0: + print(f"ERROR: Expected no forecast.solar calls when Open-Meteo succeeds, got {len(forecast_calls)}") + failed = True + + if f"sensor.{test_api.mock_base.prefix}_pv_today" not in test_api.dashboard_items: + print("ERROR: Expected pv_today sensor to be published from Open-Meteo primary") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure(my_predbat): + """ + When forecast_solar_open_meteo_first is True and Open-Meteo returns no data, + fetch_pv_forecast falls back to forecast.solar. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + # Open-Meteo fails + test_api.set_mock_response("api.open-meteo.com", {"error": "server error"}, 500) + test_api.set_mock_response("ensemble-api.open-meteo.com", {"error": "server error"}, 500) + # forecast.solar succeeds + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]] + if len(forecast_calls) == 0: + print("ERROR: Expected forecast.solar fallback call when Open-Meteo fails, got none") + failed = True + + if f"sensor.{test_api.mock_base.prefix}_pv_today" not in test_api.dashboard_items: + print("ERROR: Expected pv_today sensor to be published after forecast.solar fallback") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_ignored_when_unset(my_predbat): + """ + When forecast_solar_open_meteo_first is False the existing ordering is unchanged: + forecast.solar is primary and Open-Meteo is not called. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_ignored_when_unset") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = False + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]] + if len(forecast_calls) == 0: + print("ERROR: Expected forecast.solar to remain primary when open_meteo_first is False") + failed = True + + open_meteo_calls = [r for r in test_api.request_log if "open-meteo.com" in r["url"]] + if len(open_meteo_calls) != 0: + print(f"ERROR: Expected no Open-Meteo calls when open_meteo_first is False, got {len(open_meteo_calls)}") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south(my_predbat): + """ + A forecast_solar entry with azimuth_zero_south True must reach the Open-Meteo request + with the azimuth unconverted. A regression here would silently mis-orient every array. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 54.81306, "longitude": -1.38647, "declination": 32, "azimuth": 85, "azimuth_zero_south": True, "kwp": 6.44}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + test_api.set_mock_response( + "api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00"], + "global_tilted_irradiance": [500.0, 600.0], + "temperature_2m": [25.0, 25.0], + "wind_speed_10m": [1.0, 1.0], + } + }, + ) + test_api.set_mock_response( + "ensemble-api.open-meteo.com", + {"hourly": {"time": ["2025-06-15T12:00", "2025-06-15T13:00"], "global_tilted_irradiance_member01": [400.0, 480.0]}}, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + forecast_urls = [r["url"] for r in test_api.request_log if "api.open-meteo.com" in r["url"]] + if not forecast_urls: + print("ERROR: Expected an Open-Meteo forecast request, got none") + failed = True + for url in forecast_urls: + if "azimuth=85" not in url: + print(f"ERROR: Expected azimuth=85 (unconverted) in Open-Meteo URL, got {url}") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_logs_source_change(my_predbat): + """ + Switching the active forecast source emits a warning so the 7-day PV calibration + settling period is visible in the log rather than silently skewing the scaling factor. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_logs_source_change") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.open_meteo_forecast_max_age = 1.0 + test_api.set_mock_response( + "api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00"], + "global_tilted_irradiance": [500.0, 600.0], + "temperature_2m": [25.0, 25.0], + "wind_speed_10m": [1.0, 1.0], + } + }, + ) + test_api.set_mock_response( + "ensemble-api.open-meteo.com", + {"hourly": {"time": ["2025-06-15T12:00", "2025-06-15T13:00"], "global_tilted_irradiance_member01": [400.0, 480.0]}}, + ) + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + # MockBase.log only prints, so capture messages by replacing the copied log reference. + # ComponentBase copies base.log onto the component, so this override is local to the test. + captured = [] + + def capture_log(message, quiet=True): + """Capture a log message emitted by SolarAPI.""" + captured.append(message) + + test_api.solar.log = capture_log + + # First run on forecast.solar establishes the stored source, no warning expected + test_api.solar.forecast_solar_open_meteo_first = False + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + changed_first_run = [m for m in captured if "forecast source changed" in m] + if changed_first_run: + print(f"ERROR: Did not expect a source change warning on the first run, got {changed_first_run}") + failed = True + + # Second run flips to Open-Meteo - a warning is expected + captured.clear() + test_api.solar.forecast_solar_open_meteo_first = True + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + changed_second_run = [m for m in captured if "forecast source changed" in m] + if not changed_second_run: + print("ERROR: Expected a source change warning after switching to Open-Meteo, got none") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_source_change_warning_steady_state(my_predbat): + """ + Two consecutive identical fetch cycles must not repeat the source-change warning. + fetch_pv_forecast runs every plan interval, so a warning that re-fires on every steady-state + cycle instead of only on a genuine configuration change would flood the log and the web UI's + Warnings tab. + """ + print(" - test_fetch_pv_forecast_source_change_warning_steady_state") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = False + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + captured = [] + + def capture_log(message, quiet=True): + """Capture a log message emitted by SolarAPI.""" + captured.append(message) + + test_api.solar.log = capture_log + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + captured.clear() + run_async(test_api.solar.fetch_pv_forecast()) + + changed_second_run = [m for m in captured if "forecast source changed" in m] + if changed_second_run: + print(f"ERROR: Expected no source-change warning on a second identical cycle, got {changed_second_run}") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_transient_fallback_no_warning(my_predbat): + """ + A same-cycle fallback (Open-Meteo fails, Forecast.solar covers it) must not warn about a + source change: the configured/intended source is still Open-Meteo, only the data used this + cycle happened to come from Forecast.solar. Warning here would be a false alarm for every + transient Open-Meteo blip. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_transient_fallback_no_warning") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + + # Simulate a prior cycle having already settled on Open-Meteo as the configured source, + # so this run exercises the same-cycle-fallback path rather than the no-previous-source + # case, which would never warn regardless of the fix. + run_async(test_api.solar.storage.save("solcast", "active_forecast_source", {"source": "open_meteo"}, format="json", expiry=None)) + + # Open-Meteo fails this cycle only + test_api.set_mock_response("api.open-meteo.com", {"error": "server error"}, 500) + test_api.set_mock_response("ensemble-api.open-meteo.com", {"error": "server error"}, 500) + # Forecast.solar fallback succeeds + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + captured = [] + + def capture_log(message, quiet=True): + """Capture a log message emitted by SolarAPI.""" + captured.append(message) + + test_api.solar.log = capture_log + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + changed = [m for m in captured if "forecast source changed" in m] + if changed: + print(f"ERROR: A same-cycle fallback must not warn about a source change, got {changed}") + failed = True + + finally: + test_api.cleanup() + + return failed + + def test_fetch_pv_forecast_ha_sensors(my_predbat): """ Integration test: fetch_pv_forecast using HA sensors (Solcast integration). @@ -3708,6 +4110,13 @@ def run_solcast_tests(my_predbat): failed |= test_fetch_pv_forecast_forecast_solar(my_predbat) failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_on_failure(my_predbat) failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_used_when_set(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_ignored_when_unset(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_logs_source_change(my_predbat) + failed |= test_fetch_pv_forecast_source_change_warning_steady_state(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_transient_fallback_no_warning(my_predbat) failed |= test_fetch_pv_forecast_ha_sensors(my_predbat) # 15-minute resolution tests diff --git a/docs/apps-yaml.md b/docs/apps-yaml.md index b3940b5e5..aab5c24c2 100644 --- a/docs/apps-yaml.md +++ b/docs/apps-yaml.md @@ -1598,6 +1598,38 @@ When the fallback is active, Predbat derives the Open-Meteo request from the sam forecast_solar_open_meteo_backup: true ``` +### Using Open-Meteo as the primary source + +Setting `forecast_solar_open_meteo_first: true` reverses the order: Predbat fetches from Open-Meteo +first and only calls Forecast.solar if Open-Meteo returns no data. Your existing `forecast_solar` +per-array entries (latitude, longitude, postcode, declination, azimuth, kwp, efficiency) are reused +as-is. If you also have an `open_meteo_forecast` section, that is used instead, which lets you apply +Open-Meteo-specific options such as `shading_factors`. + +```yaml + forecast_solar: + - postcode: SW1A 2AB + kwp: 3 + azimuth: 45 + declination: 45 + forecast_solar_open_meteo_first: true +``` + +While Open-Meteo is succeeding, Forecast.solar is not called at all, so no Forecast.solar API quota +is consumed. + +Note that `forecast_solar_max_age` is not reused while this flag is set — it only applies to the +Forecast.solar path. The refresh interval instead comes from `open_meteo_forecast_max_age` +(default 4 hours). + +If `forecast_solar_open_meteo_backup` is also set to true, it has no effect: `forecast_solar_open_meteo_first` +already makes Open-Meteo the primary source, so there is nothing left for the backup setting to do. + +Note that PV calibration compares the last seven days of recorded forecasts against actual +generation. After changing the source, that history still holds values from the previous source, so +the calibration scaling factor takes up to seven days to settle. Predbat logs a warning when the +source changes. Do not judge the accuracy of the new source until the settling period has passed. + ## Open-Meteo Solar Forecast [Open-Meteo](https://open-meteo.com/) is a free, open-source weather API that provides solar irradiance forecasts with no API key required. diff --git a/docs/install.md b/docs/install.md index 9db04dcac..ef1dee2fd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -164,6 +164,14 @@ The optional `azimuth_zero_south` (default False) can be set to True if you pref forecast_solar_max_age: 4 ``` +Setting `forecast_solar_open_meteo_first: true` makes Predbat use Open-Meteo as the primary forecast +source and fall back to Forecast.solar only if Open-Meteo returns no data. Your existing +`forecast_solar` per-array settings (postcode, latitude, longitude, azimuth, declination, kwp, +efficiency) are reused unchanged, but `forecast_solar_max_age` is not — the refresh interval instead +comes from `open_meteo_forecast_max_age` while this flag is set. See the +[Using Open-Meteo as the primary source](apps-yaml.md#using-open-meteo-as-the-primary-source) section +of the apps.yaml documentation for details. + or you can set longitude and latitude if you are not in the UK or postcode does not work: ```yaml diff --git a/docs/superpowers/plans/2026-07-30-open-meteo-primary-source.md b/docs/superpowers/plans/2026-07-30-open-meteo-primary-source.md new file mode 100644 index 000000000..bf25c1e85 --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-open-meteo-primary-source.md @@ -0,0 +1,683 @@ +# Open-Meteo Primary Source Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a single boolean, `forecast_solar_open_meteo_first`, that makes Predbat fetch solar forecasts from Open-Meteo first and fall back to Forecast.solar, reusing the existing `forecast_solar` configuration unchanged. + +**Architecture:** A new branch is added ahead of the existing `self.forecast_solar` branch in `SolarAPI.fetch_pv_forecast()`. It mirrors the existing Open-Meteo backup path in reverse: Open-Meteo is called first, and Forecast.solar is called only when Open-Meteo returns no data. The `forecast_solar` config entries are passed straight to `download_open_meteo_data()` because both download paths read the same keys and apply `azimuth_zero_south` identically. + +**Tech Stack:** Python 3, asyncio, aiohttp. Tests use the project's own `TestSolarAPI` harness in `apps/predbat/tests/test_solcast.py` with mocked HTTP; no pytest. + +## Global Constraints + +- Line length: 256 chars (Black), 250 chars (Flake8). +- Docstrings required on every function and class (`interrogate`, 100% coverage). The new helper method must have one. +- Spell checking is British English via CSpell. `docs/superpowers/` is excluded, but `docs/apps-yaml.md` and `docs/install.md` are **not** — new words there must be added to `.cspell/custom-dictionary-workspace.txt`. +- Variable naming: `lower_case_with_underscores`. +- Tests are run from the `coverage/` directory. Always redirect test output to a file and grep the file afterwards; never pipe directly to grep. +- The existing `forecast_solar_open_meteo_backup` setting must not change in type or behaviour. +- Do not modify the caps in `pv_calibration()` or the `kwp * efficiency` value sent to Forecast.solar. Both are explicitly out of scope in the spec. + +## Reference: how to run the tests + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all --test solcast > /tmp/solcast_out.txt 2>&1 +grep -E "ERROR|FAIL|Passed|failed" /tmp/solcast_out.txt +``` + +A passing run prints no `ERROR:` lines from the solcast tests. Each test function returns a `failed` boolean; `run_solcast_tests` ORs them together. + +--- + +### Task 1: Add the setting and reverse the source order + +**Files:** +- Modify: `apps/predbat/config.py:2367` (add schema entry) +- Modify: `apps/predbat/components.py:109` (add component arg) +- Modify: `apps/predbat/solcast.py:47-76` (add `initialize()` parameter and assignment) +- Modify: `apps/predbat/solcast.py:1218-1226` (reverse the source order) +- Test: `apps/predbat/tests/test_solcast.py` (harness update at line 171, plus four new tests) + +**Interfaces:** +- Consumes: `SolarAPI.download_open_meteo_data(configs=...)` returning `(sorted_data, max_kwh)`; `SolarAPI.download_forecast_solar_data()` returning `(sorted_data, max_kwh)`. +- Produces: `self.forecast_solar_open_meteo_first` (bool, default `False`) on the `SolarAPI` instance. Task 2 reads nothing from this task beyond the branch structure it edits. + +- [ ] **Step 1: Write the four failing tests** + +Add these four functions to `apps/predbat/tests/test_solcast.py`, immediately after `test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success` (which ends around line 1822, just before `def test_fetch_pv_forecast_ha_sensors`). + +```python +def test_fetch_pv_forecast_open_meteo_first_used_when_set(my_predbat): + """ + When forecast_solar_open_meteo_first is True, Open-Meteo is used as the primary + source and forecast.solar is not called at all. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_used_when_set") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + # Both sources would succeed - Open-Meteo must win and forecast.solar must not be called + test_api.set_mock_response( + "api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"], + "global_tilted_irradiance": [500.0, 600.0, 550.0], + "temperature_2m": [25.0, 25.0, 25.0], + "wind_speed_10m": [1.0, 1.0, 1.0], + } + }, + ) + test_api.set_mock_response( + "ensemble-api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"], + "global_tilted_irradiance_member01": [400.0, 480.0, 440.0], + } + }, + ) + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + open_meteo_calls = [r for r in test_api.request_log if "open-meteo.com" in r["url"]] + if len(open_meteo_calls) == 0: + print("ERROR: Expected Open-Meteo API call when open_meteo_first is set, got none") + failed = True + + forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]] + if len(forecast_calls) != 0: + print(f"ERROR: Expected no forecast.solar calls when Open-Meteo succeeds, got {len(forecast_calls)}") + failed = True + + if f"sensor.{test_api.mock_base.prefix}_pv_today" not in test_api.dashboard_items: + print("ERROR: Expected pv_today sensor to be published from Open-Meteo primary") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure(my_predbat): + """ + When forecast_solar_open_meteo_first is True and Open-Meteo returns no data, + fetch_pv_forecast falls back to forecast.solar. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + # Open-Meteo fails + test_api.set_mock_response("api.open-meteo.com", {"error": "server error"}, 500) + test_api.set_mock_response("ensemble-api.open-meteo.com", {"error": "server error"}, 500) + # forecast.solar succeeds + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]] + if len(forecast_calls) == 0: + print("ERROR: Expected forecast.solar fallback call when Open-Meteo fails, got none") + failed = True + + if f"sensor.{test_api.mock_base.prefix}_pv_today" not in test_api.dashboard_items: + print("ERROR: Expected pv_today sensor to be published after forecast.solar fallback") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_ignored_when_unset(my_predbat): + """ + When forecast_solar_open_meteo_first is False the existing ordering is unchanged: + forecast.solar is primary and Open-Meteo is not called. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_ignored_when_unset") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.forecast_solar_open_meteo_first = False + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]] + if len(forecast_calls) == 0: + print("ERROR: Expected forecast.solar to remain primary when open_meteo_first is False") + failed = True + + open_meteo_calls = [r for r in test_api.request_log if "open-meteo.com" in r["url"]] + if len(open_meteo_calls) != 0: + print(f"ERROR: Expected no Open-Meteo calls when open_meteo_first is False, got {len(open_meteo_calls)}") + failed = True + + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south(my_predbat): + """ + A forecast_solar entry with azimuth_zero_south True must reach the Open-Meteo request + with the azimuth unconverted. A regression here would silently mis-orient every array. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 54.81306, "longitude": -1.38647, "declination": 32, "azimuth": 85, "azimuth_zero_south": True, "kwp": 6.44}] + test_api.solar.forecast_solar_open_meteo_first = True + test_api.solar.open_meteo_forecast_max_age = 1.0 + test_api.set_mock_response( + "api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00"], + "global_tilted_irradiance": [500.0, 600.0], + "temperature_2m": [25.0, 25.0], + "wind_speed_10m": [1.0, 1.0], + } + }, + ) + test_api.set_mock_response( + "ensemble-api.open-meteo.com", + {"hourly": {"time": ["2025-06-15T12:00", "2025-06-15T13:00"], "global_tilted_irradiance_member01": [400.0, 480.0]}}, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + forecast_urls = [r["url"] for r in test_api.request_log if "api.open-meteo.com" in r["url"]] + if not forecast_urls: + print("ERROR: Expected an Open-Meteo forecast request, got none") + failed = True + for url in forecast_urls: + if "azimuth=85" not in url: + print(f"ERROR: Expected azimuth=85 (unconverted) in Open-Meteo URL, got {url}") + failed = True + + finally: + test_api.cleanup() + + return failed +``` + +Register all four in `run_solcast_tests`. Find the block near line 3709 and add them immediately after the existing `..._backup_not_used_on_success` line: + +```python + failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_used_when_set(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_ignored_when_unset(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south(my_predbat) +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all --test solcast > /tmp/solcast_step2.txt 2>&1 +grep -E "ERROR|Traceback|AttributeError" /tmp/solcast_step2.txt +``` + +Expected: failures. `test_api.solar.forecast_solar_open_meteo_first = True` sets an attribute that `fetch_pv_forecast` never reads, so Open-Meteo is not called and `test_..._used_when_set` reports `ERROR: Expected Open-Meteo API call when open_meteo_first is set, got none`. + +- [ ] **Step 3: Add the schema entry** + +In `apps/predbat/config.py`, find line 2367 and insert the new key directly after it: + +```python + "forecast_solar_open_meteo_backup": {"type": "boolean"}, + "forecast_solar_open_meteo_first": {"type": "boolean"}, + "open_meteo_forecast": {"type": "dict_list"}, +``` + +- [ ] **Step 4: Add the component arg** + +In `apps/predbat/components.py`, insert directly after line 109: + +```python + "forecast_solar_open_meteo_backup": {"required": False, "config": "forecast_solar_open_meteo_backup", "default": False}, + "forecast_solar_open_meteo_first": {"required": False, "config": "forecast_solar_open_meteo_first", "default": False}, +``` + +- [ ] **Step 5: Add the initialize() parameter** + +In `apps/predbat/solcast.py`, add the parameter to `SolarAPI.initialize()` directly after `forecast_solar_open_meteo_backup,` (line 56): + +```python + forecast_solar_open_meteo_backup, + forecast_solar_open_meteo_first, +``` + +And the assignment directly after line 73: + +```python + self.forecast_solar_open_meteo_backup = forecast_solar_open_meteo_backup + self.forecast_solar_open_meteo_first = forecast_solar_open_meteo_first +``` + +- [ ] **Step 6: Update the test harness** + +`TestSolarAPI.__init__` calls `initialize()` with keyword arguments, so it must pass the new one or every solcast test raises `TypeError`. In `apps/predbat/tests/test_solcast.py`, add directly after line 171: + +```python + forecast_solar_open_meteo_backup=False, + forecast_solar_open_meteo_first=False, +``` + +- [ ] **Step 7: Reverse the source order** + +In `apps/predbat/solcast.py`, replace lines 1218-1226 (the `if self.forecast_solar:` block through the existing backup fallback) with: + +```python + if self.forecast_solar and self.forecast_solar_open_meteo_first: + self.log("SolarAPI: Obtaining solar forecast from Open-Meteo API (primary, Forecast Solar fallback)") + primary_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar + pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=primary_configs) + divide_by = 30.0 + create_pv10 = True + if not pv_forecast_data: + self.log("Warn: SolarAPI: Open-Meteo returned no data, falling back to Forecast Solar") + pv_forecast_data, max_kwh = await self.download_forecast_solar_data() + elif self.forecast_solar: + self.log("SolarAPI: Obtaining solar forecast from Forecast Solar API") + pv_forecast_data, max_kwh = await self.download_forecast_solar_data() + divide_by = 30.0 + create_pv10 = True + if not pv_forecast_data and self.forecast_solar_open_meteo_backup: + self.log("SolarAPI: Forecast Solar returned no data, falling back to Open-Meteo backup") + backup_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar + pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=backup_configs) +``` + +Leave the `elif self.open_meteo_forecast:` branch and everything below it untouched. + +- [ ] **Step 8: Run the tests to verify they pass** + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all --test solcast > /tmp/solcast_step8.txt 2>&1 +grep -E "ERROR|Traceback|FAIL" /tmp/solcast_step8.txt +``` + +Expected: no output from grep. If `test_..._ignored_when_unset` fails, the `elif` ordering is wrong — the `open_meteo_first` branch must be checked before the plain `forecast_solar` branch. + +- [ ] **Step 9: Run pre-commit** + +```bash +cd /Users/treforsouthwell/source/batpred +source coverage/venv/bin/activate +pre-commit run --files apps/predbat/solcast.py apps/predbat/config.py apps/predbat/components.py apps/predbat/tests/test_solcast.py +``` + +Expected: all hooks Passed. Black may reformat; if it does, re-stage the files. + +- [ ] **Step 10: Commit** + +```bash +git add apps/predbat/solcast.py apps/predbat/config.py apps/predbat/components.py apps/predbat/tests/test_solcast.py +git commit -m "feat(solar): add forecast_solar_open_meteo_first to use Open-Meteo as primary source" +``` + +--- + +### Task 2: Warn when the active forecast source changes + +**Files:** +- Modify: `apps/predbat/solcast.py` (add `log_source_change()` helper; set `active_source` in each branch of `fetch_pv_forecast`; call the helper before the `if pv_forecast_data:` block at line 1267) +- Test: `apps/predbat/tests/test_solcast.py` (one new test) + +**Interfaces:** +- Consumes: `self.storage` (may be `None`), `self.storage.load(module, filename)` and `self.storage.save(module, filename, data, format=, expiry=)`, both async. +- Produces: `async SolarAPI.log_source_change(source)` returning `None`. Persists `{"source": }` under module `"solcast"`, filename `"active_forecast_source"`. + +Source names used: `"open_meteo"`, `"forecast_solar"`, `"solcast"`, `"ha_sensors"`. + +- [ ] **Step 1: Write the failing test** + +Add to `apps/predbat/tests/test_solcast.py`, after `test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south`: + +```python +def test_fetch_pv_forecast_open_meteo_first_logs_source_change(my_predbat): + """ + Switching the active forecast source emits a warning so the 7-day PV calibration + settling period is visible in the log rather than silently skewing the scaling factor. + """ + print(" - test_fetch_pv_forecast_open_meteo_first_logs_source_change") + failed = False + + test_api = create_test_solar_api() + try: + test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.open_meteo_forecast_max_age = 1.0 + test_api.set_mock_response( + "api.open-meteo.com", + { + "hourly": { + "time": ["2025-06-15T12:00", "2025-06-15T13:00"], + "global_tilted_irradiance": [500.0, 600.0], + "temperature_2m": [25.0, 25.0], + "wind_speed_10m": [1.0, 1.0], + } + }, + ) + test_api.set_mock_response( + "ensemble-api.open-meteo.com", + {"hourly": {"time": ["2025-06-15T12:00", "2025-06-15T13:00"], "global_tilted_irradiance_member01": [400.0, 480.0]}}, + ) + test_api.set_mock_response( + "forecast.solar", + { + "result": {"watts": {"2025-06-15T12:00:00+0000": 500, "2025-06-15T12:30:00+0000": 600}}, + "message": {"info": {"time": "2025-06-15T11:30:00+0000"}}, + }, + 200, + ) + + def create_mock_session(*args, **kwargs): + """Create a mock aiohttp session.""" + return test_api.mock_aiohttp_session() + + # MockBase.log only prints, so capture messages by replacing the copied log reference. + # ComponentBase copies base.log onto the component, so this override is local to the test. + captured = [] + + def capture_log(message, quiet=True): + """Capture a log message emitted by SolarAPI.""" + captured.append(message) + + test_api.solar.log = capture_log + + # First run on forecast.solar establishes the stored source, no warning expected + test_api.solar.forecast_solar_open_meteo_first = False + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + changed_first_run = [m for m in captured if "forecast source changed" in m] + if changed_first_run: + print(f"ERROR: Did not expect a source change warning on the first run, got {changed_first_run}") + failed = True + + # Second run flips to Open-Meteo - a warning is expected + captured.clear() + test_api.solar.forecast_solar_open_meteo_first = True + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + changed_second_run = [m for m in captured if "forecast source changed" in m] + if not changed_second_run: + print("ERROR: Expected a source change warning after switching to Open-Meteo, got none") + failed = True + + finally: + test_api.cleanup() + + return failed +``` + +Register it in `run_solcast_tests` after the four added in Task 1: + +```python + failed |= test_fetch_pv_forecast_open_meteo_first_logs_source_change(my_predbat) +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all --test solcast > /tmp/solcast_t2_step3.txt 2>&1 +grep -E "ERROR|Traceback" /tmp/solcast_t2_step3.txt +``` + +Expected: `ERROR: Expected a source change warning after switching to Open-Meteo, got none`. + +- [ ] **Step 3: Add the helper method** + +In `apps/predbat/solcast.py`, add this method directly above `async def fetch_pv_forecast(self):` (line 1204): + +```python + async def log_source_change(self, source): + """Warn when the active solar forecast source changes so the PV calibration settling period is visible.""" + if not self.storage: + return + stored = await self.storage.load("solcast", "active_forecast_source") + previous = stored.get("source") if isinstance(stored, dict) else None + if previous == source: + return + if previous: + self.log("Warn: SolarAPI: Solar forecast source changed from {} to {}, PV calibration will settle over the next 7 days".format(previous, source)) + await self.storage.save("solcast", "active_forecast_source", {"source": source}, format="json", expiry=None) +``` + +- [ ] **Step 4: Record the active source in each branch** + +In `fetch_pv_forecast()`, add `active_source = "..."` assignments. In the `open_meteo_first` branch added in Task 1: + +```python + if self.forecast_solar and self.forecast_solar_open_meteo_first: + self.log("SolarAPI: Obtaining solar forecast from Open-Meteo API (primary, Forecast Solar fallback)") + primary_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar + pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=primary_configs) + divide_by = 30.0 + create_pv10 = True + active_source = "open_meteo" + if not pv_forecast_data: + self.log("Warn: SolarAPI: Open-Meteo returned no data, falling back to Forecast Solar") + pv_forecast_data, max_kwh = await self.download_forecast_solar_data() + active_source = "forecast_solar" +``` + +In the existing `elif self.forecast_solar:` branch: + +```python + active_source = "forecast_solar" + if not pv_forecast_data and self.forecast_solar_open_meteo_backup: + self.log("SolarAPI: Forecast Solar returned no data, falling back to Open-Meteo backup") + backup_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar + pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=backup_configs) + active_source = "open_meteo" +``` + +In the `elif self.open_meteo_forecast:` branch add `active_source = "open_meteo"`; in the `elif self.solcast_host and self.solcast_api_key:` branch add `active_source = "solcast"`; in the final `else:` (HA sensors) branch add `active_source = "ha_sensors"` immediately after `using_ha_data = True`. + +- [ ] **Step 5: Call the helper** + +Insert immediately before the `if pv_forecast_data:` line (line 1267 before this task's edits), after the whole if/elif chain has completed: + +```python + await self.log_source_change(active_source) + + if pv_forecast_data: +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all --test solcast > /tmp/solcast_t2_step6.txt 2>&1 +grep -E "ERROR|Traceback|FAIL" /tmp/solcast_t2_step6.txt +``` + +Expected: no output from grep. + +- [ ] **Step 7: Run the full quick suite** + +`fetch_pv_forecast` is on the main path, so check nothing else regressed: + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all --quick > /tmp/quick_out.txt 2>&1 +grep -E "ERROR|FAIL|Traceback" /tmp/quick_out.txt +tail -5 /tmp/quick_out.txt +``` + +Expected: no errors. Note: if the suite hangs, a locally running Predbat may be holding port 5052 — stop it and re-run. + +- [ ] **Step 8: Run pre-commit and commit** + +```bash +cd /Users/treforsouthwell/source/batpred +source coverage/venv/bin/activate +pre-commit run --files apps/predbat/solcast.py apps/predbat/tests/test_solcast.py +git add apps/predbat/solcast.py apps/predbat/tests/test_solcast.py +git commit -m "feat(solar): warn when the active solar forecast source changes" +``` + +--- + +### Task 3: Document the setting + +**Files:** +- Modify: `docs/apps-yaml.md:1584-1598` (the "Open-Meteo backup for Forecast.solar" section) +- Modify: `docs/install.md` (the matching backup section, around line 155) + +**Interfaces:** +- Consumes: the setting name `forecast_solar_open_meteo_first` from Task 1. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Extend the apps-yaml.md section** + +In `docs/apps-yaml.md`, append to the "Open-Meteo backup for Forecast.solar" section, after the existing example block: + +````markdown +### Using Open-Meteo as the primary source + +Setting `forecast_solar_open_meteo_first: true` reverses the order: Predbat fetches from Open-Meteo +first and only calls Forecast.solar if Open-Meteo returns no data. Your existing `forecast_solar` +entries are reused as-is, so no other configuration changes are needed. If you also have an +`open_meteo_forecast` section, that is used instead, which lets you apply Open-Meteo-specific +options such as `shading_factors`. + +```yaml + forecast_solar: + - postcode: SW1A 2AB + kwp: 3 + azimuth: 45 + declination: 45 + forecast_solar_open_meteo_first: true +``` + +While Open-Meteo is succeeding, Forecast.solar is not called at all, so no Forecast.solar API quota +is consumed. + +Note that PV calibration compares the last seven days of recorded forecasts against actual +generation. After changing the source, that history still holds values from the previous source, so +the calibration scaling factor takes up to seven days to settle. Predbat logs a warning when the +source changes. Do not judge the accuracy of the new source until the settling period has passed. +```` + +- [ ] **Step 2: Add the matching note to install.md** + +In `docs/install.md`, after the existing `azimuth_zero_south` paragraph around line 155, add: + +```markdown +Setting `forecast_solar_open_meteo_first: true` makes Predbat use Open-Meteo as the primary forecast +source and fall back to Forecast.solar only if Open-Meteo returns no data. Your existing +`forecast_solar` settings are reused unchanged. See the Open-Meteo section in +[apps-yaml.md](apps-yaml.md) for details. +``` + +- [ ] **Step 3: Run pre-commit** + +Both files are spell-checked and markdown-linted (unlike `docs/superpowers/`). + +```bash +cd /Users/treforsouthwell/source/batpred +source coverage/venv/bin/activate +pre-commit run --files docs/apps-yaml.md docs/install.md +``` + +Expected: all Passed. If CSpell rejects a word, add it to `.cspell/custom-dictionary-workspace.txt`, then re-run pre-commit and re-stage — the dictionary file is auto-sorted on commit. + +- [ ] **Step 4: Commit** + +```bash +git add docs/apps-yaml.md docs/install.md +git commit -m "docs(solar): document forecast_solar_open_meteo_first" +``` + +--- + +## Verification + +After all three tasks: + +```bash +cd /Users/treforsouthwell/source/batpred/coverage +source venv/bin/activate +./run_all > /tmp/full_out.txt 2>&1 +grep -E "ERROR|FAIL|Traceback" /tmp/full_out.txt +tail -20 /tmp/full_out.txt +``` + +Expected: the full suite passes. + +Manual check on the target site: set `forecast_solar_open_meteo_first: true`, restart, and confirm the +log shows `SolarAPI: Obtaining solar forecast from Open-Meteo API (primary, Forecast Solar fallback)`, +a source change warning, and no `https://api.forecast.solar/` request. diff --git a/docs/superpowers/specs/2026-07-30-open-meteo-primary-source-design.md b/docs/superpowers/specs/2026-07-30-open-meteo-primary-source-design.md new file mode 100644 index 000000000..229de2b1d --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-open-meteo-primary-source-design.md @@ -0,0 +1,148 @@ +# Open-Meteo as primary solar source with Forecast.solar fallback + +Date: 2026-07-30 + +## Motivation + +Investigation of a customer report ("solar constantly under reports") found that Forecast.solar +carried a consistent low bias for that site: over seven logged days it forecast 127.2 kWh against +168.7 kWh actually generated, low on seven days out of seven, giving PV calibration an +`average_day_scaling` of 1.32. + +The investigation ruled out every Predbat-side cause: + +- The `watts` to kWh integration in `download_forecast_solar_data()` reproduces Forecast.solar's own + `watt_hours_day` to within 0.35%, so no energy is lost in parsing. +- The caps in `pv_calibration()` do not clip the planner's data. They apply only to + `pv_estimateCL`/`pv_estimate10`/`pv_estimate90`, which annotate the published sensor attributes; + `solcast.py` returns the uncapped `pv_forecast_minute_adjusted` to the planner. +- The customer's configuration is correct. Reconstructing the actual generation profile from the + logged slot adjustments fits the configured azimuth of 85 (WSW) far better than south or east + (SSE 0.00067 against 0.00586 and 0.01920). Open-Meteo GTI at the same tilt, azimuth and kWp + predicts 182.9 kWh across the same seven days against 168.7 kWh actual, confirming the declared + 6.44 kWp is right. + +Open-Meteo is therefore worth evaluating as the primary source for affected sites. Predbat can +already fetch Open-Meteo, but only as a failure fallback via `forecast_solar_open_meteo_backup`, +which fires only when Forecast.solar returns no data. A biased-but-successful response never +triggers it. + +This design adds a single setting that reverses the order, so a site can be switched to Open-Meteo +without restructuring its `forecast_solar` configuration. + +Note on cost: the Open-Meteo path calls the Ensemble API for P10. Ensemble is available on the free +tier, rate limited, so evaluating a small number of customers needs no paid plan. + +## The setting + +A new top-level boolean, `forecast_solar_open_meteo_first`, defaulting to `False`: + +```yaml +forecast_solar: + - latitude: 54.81306 + longitude: -1.38647 + declination: 32 + azimuth: 85 + azimuth_zero_south: True + kwp: 6.44 + api_key: xxxx + +forecast_solar_open_meteo_first: True +``` + +The existing `forecast_solar_open_meteo_backup` is unchanged in both type and behaviour. When +`forecast_solar_open_meteo_first` is set it implies the reverse fallback, so it works standalone and +`forecast_solar_open_meteo_backup` becomes irrelevant. + +Rationale for a sibling boolean over extending the existing flag into a mode: the existing setting is +declared `{"type": "boolean"}` in `APPS_SCHEMA` and is documented as a boolean in two places. +Extending it to boolean-or-string requires back-compat handling for no functional gain. A separate +flag is also easier to toggle per customer during an evaluation. + +## Behaviour + +`fetch_pv_forecast()` gains a branch ahead of the existing `self.forecast_solar` branch: + +```python +if self.forecast_solar and self.forecast_solar_open_meteo_first: + primary_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar + pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=primary_configs) + divide_by = 30.0 + create_pv10 = True + if not pv_forecast_data: + pv_forecast_data, max_kwh = await self.download_forecast_solar_data() +elif self.forecast_solar: + # existing branch, unchanged +``` + +The config-selection line mirrors the existing backup path: an explicit `open_meteo_forecast` section +wins if present, so Open-Meteo-specific options such as `shading_factors` remain available; +otherwise the `forecast_solar` entries are reused directly. + +Forecast.solar is not called at all while Open-Meteo succeeds. This is deliberate: it keeps the +comparison clean and stops consuming the site's Forecast.solar quota. + +### Why config reuse needs no translation + +`download_open_meteo_data()` and `download_forecast_solar_data()` read the same keys (`latitude`, +`longitude`, `postcode`, `declination`, `azimuth`, `kwp`, `efficiency`) and apply `azimuth_zero_south` +identically, both calling `convert_azimuth()` only when the flag is absent or false. Both return +`max_kwh` as `kwp * efficiency`. An `api_key` present in a `forecast_solar` entry is ignored by the +Open-Meteo path. `shading_factors` is optional and simply absent when reusing `forecast_solar` +entries. + +The azimuth convention is the one silent-failure risk in this reuse: a mismatch would mis-orient +every array without raising an error, so it is covered by a dedicated test. + +### Fallback trigger + +`not pv_forecast_data`, the same condition the existing backup path uses. A biased-but-successful +Open-Meteo response does not fall back, matching the existing semantics. + +## Calibration during a source change + +PV calibration derives `past_day_forecast` from up to seven days of `sensor.predbat_pv_forecast_h0` +history. Immediately after the setting is flipped that history still holds Forecast.solar values, +so `average_day_scaling` blends the two sources until the window rolls over. + +The chosen handling is to accept the settling period rather than reset or partition the history. +Calibration self-corrects within seven days, and resetting would instead disable calibration +entirely until three valid days accumulate. + +To make the transition visible rather than silent, the active source name is persisted through the +existing Storage abstraction and a warning is logged when it differs from the previous run, +stating that calibration will settle over the following seven days. + +Practical consequence: an A/B comparison should not be judged until seven days after the switch. + +## Testing + +Mirroring the two existing backup tests in `tests/test_solcast.py`: + +- `test_fetch_pv_forecast_open_meteo_first_used_when_set` — Open-Meteo returns data; assert it is + used and `download_forecast_solar_data` is never called. +- `test_fetch_pv_forecast_open_meteo_first_falls_back_on_failure` — Open-Meteo returns nothing; + assert Forecast.solar data is used. +- `test_fetch_pv_forecast_open_meteo_first_ignored_when_unset` — flag off; assert existing ordering + is intact. +- `test_fetch_pv_forecast_open_meteo_first_preserves_azimuth_zero_south` — a `forecast_solar` entry + with `azimuth_zero_south: True` reaches the Open-Meteo request with the azimuth unconverted. +- `test_fetch_pv_forecast_open_meteo_first_logs_source_change` — source change emits the warning. + +## Documentation + +Extend the existing "Open-Meteo backup for Forecast.solar" section in `docs/apps-yaml.md` and the +matching section in `docs/install.md`, covering the new setting, the reversed fallback, the fact +that Forecast.solar is not called while Open-Meteo succeeds, and the seven-day calibration settling +period. + +## Out of scope + +- The cap inconsistency where published `pv_estimateCL`/`10`/`90` attributes are capped to + nameplate/observed peak while the planner's data is not. Worth fixing separately; it is the most + likely explanation for a user reporting a forecast apparently pinned near a fixed ceiling. +- The possible double-derate where `kwp * efficiency` is sent to Forecast.solar, which applies its + own internal system losses. Unverified, roughly 5%, and affects only the Forecast.solar path. +- Any change to `forecast_solar_open_meteo_backup`. +- Reducing Open-Meteo request volume (coordinate rounding for cache sharing, computing GTI locally + from GHI/DNI/DHI, decoupling the Ensemble refresh cadence). Relevant only at fleet scale.