From 42dff40a84dc931d8dfcc6ae50d7cac25fab723f Mon Sep 17 00:00:00 2001 From: Andrew Robbertz <24920994+Alrobbertz@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:19:51 -0400 Subject: [PATCH 1/2] Migrate to use swxsoc from sdc_aws_utils --- lambda_function/requirements.txt | 2 +- lambda_function/src/executor/executor.py | 27 +++++++++++------------- lambda_function/tests/test_executor.py | 6 +++--- requirements.dev.txt | 2 +- ruff.toml | 9 ++++++++ 5 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 ruff.toml diff --git a/lambda_function/requirements.txt b/lambda_function/requirements.txt index 3675cd4..9f17ef1 100755 --- a/lambda_function/requirements.txt +++ b/lambda_function/requirements.txt @@ -2,4 +2,4 @@ pandas stixdcpy padre_craft @ git+https://github.com/padreSat/padre_craft.git swxsoc_reach @ git+https://github.com/swxsoc/swxsoc_reach.git -sdc_aws_utils @ git+https://github.com/swxsoc/sdc_aws_utils.git \ No newline at end of file +swxsoc @ git+https://github.com/swxsoc/swxsoc.git@main \ No newline at end of file diff --git a/lambda_function/src/executor/executor.py b/lambda_function/src/executor/executor.py index c8a2a65..13b8776 100755 --- a/lambda_function/src/executor/executor.py +++ b/lambda_function/src/executor/executor.py @@ -18,10 +18,11 @@ from astropy import units as u from astropy.time import Time, TimeDelta from astropy.timeseries import TimeSeries -from sdc_aws_utils.aws import push_science_file -from sdc_aws_utils.config import parser as science_filename_parser from swxsoc import log -from swxsoc.util import util +from swxsoc.db.timeseries import record_timeseries +from swxsoc.io.s3 import push_science_file +from swxsoc.util.grafana import create_annotation +from swxsoc.util.util import parse_science_filename from swxsoc_reach.net.udl import download_UDL_reach_window @@ -158,7 +159,7 @@ def import_stix_to_timestream() -> None: log.info( f"Received stix data from {stix_ts.time[0]} to {stix_ts.time[-1]}, {len(stix_ts)} entries" ) - util.record_timeseries(stix_ts, ts_name="solo", instrument_name="stix") + record_timeseries(stix_ts, ts_name="solo", instrument_name="stix") else: log.info("No stix data received.") @@ -227,7 +228,7 @@ def _upload_reach_file_to_s3(filepath: str) -> list[str]: os.environ["SWXSOC_MISSION"] = "swxsoc_pipeline" import swxsoc - swxsoc._reconfigure() # Updates Mission Config to use swxsoc_pipeline settings + swxsoc.reconfigure() # Updates Mission Config to use swxsoc_pipeline settings calibrated_filename = os.path.basename(filepath) destination_buckets = [ @@ -239,10 +240,10 @@ def _upload_reach_file_to_s3(filepath: str) -> list[str]: new_file_keys = [] for this_destination_bucket in destination_buckets: - # Push the file to S3 using sdc_aws_utils helper + # Push the file to S3 using swxsoc.io.s3 helper new_file_keys.append( push_science_file( - science_filename_parser=science_filename_parser, + science_filename_parser=parse_science_filename, destination_bucket=this_destination_bucket, calibrated_filename=calibrated_filename, ) @@ -381,15 +382,11 @@ def import_GOES_data_to_timestream() -> None: tsb_last = tsb.loc[last_hour : Time.now()] if len(tsa_last) > 0: - util.record_timeseries( - tsa_last, ts_name="GOES", instrument_name="goes xrsa" - ) + record_timeseries(tsa_last, ts_name="GOES", instrument_name="goes xrsa") log.info( f"GOES xrsa data import from {tsa_last.time[0]} to {tsa_last.time[-1]}, {len(tsa_last)} entries" ) - util.record_timeseries( - tsb_last, ts_name="GOES", instrument_name="goes xrsb" - ) + record_timeseries(tsb_last, ts_name="GOES", instrument_name="goes xrsb") log.info( f"GOES xrsb data import from {tsb_last.time[0]} to {tsb_last.time[-1]}, {len(tsb_last)} entries" ) @@ -434,7 +431,7 @@ def create_GOES_data_annotations() -> None: annotation_text = this_event["class"] tags = ["GOES XRS", "flare"] - util.create_annotation( + create_annotation( start_time=this_event["time"], end_time=this_event["end_time"], text=annotation_text, @@ -446,7 +443,7 @@ def create_GOES_data_annotations() -> None: ) tags.append("peak") - util.create_annotation( + create_annotation( start_time=this_event["peak_time"], text=annotation_text, tags=tags, diff --git a/lambda_function/tests/test_executor.py b/lambda_function/tests/test_executor.py index 9774271..2af089d 100755 --- a/lambda_function/tests/test_executor.py +++ b/lambda_function/tests/test_executor.py @@ -78,7 +78,7 @@ def fake_record_timeseries(*args, **kwargs): ) # Patch the record_timeseries utility function to capture its inputs for verification monkeypatch.setattr( - "src.executor.executor.util.record_timeseries", fake_record_timeseries + "src.executor.executor.record_timeseries", fake_record_timeseries ) # Invoke the function directly @@ -180,7 +180,7 @@ def fake_record_timeseries(*args, **kwargs): recorded_calls.append((args, kwargs)) monkeypatch.setattr( - "src.executor.executor.util.record_timeseries", fake_record_timeseries + "src.executor.executor.record_timeseries", fake_record_timeseries ) try: @@ -210,7 +210,7 @@ def fake_create_annotation(**kwargs): annotation_calls.append(kwargs) monkeypatch.setattr( - "src.executor.executor.util.create_annotation", fake_create_annotation + "src.executor.executor.create_annotation", fake_create_annotation ) try: diff --git a/requirements.dev.txt b/requirements.dev.txt index c9fb9ba..34d3e2f 100755 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -7,4 +7,4 @@ pandas stixdcpy padre_craft @ git+https://github.com/padreSat/padre_craft.git swxsoc_reach @ git+https://github.com/swxsoc/swxsoc_reach.git -sdc_aws_utils @ git+https://github.com/swxsoc/sdc_aws_utils.git \ No newline at end of file +swxsoc @ git+https://github.com/swxsoc/swxsoc.git@main \ No newline at end of file diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..dbe549d --- /dev/null +++ b/ruff.toml @@ -0,0 +1,9 @@ +[lint] +ignore = [ + "EXE002", # Executable file but no shebang present + "BLE001", # Do not catch blind exception: `Exception` + "TRY201", # Use `raise` without specifying exception name + "RUF028", # Invalid formatter suppression comment + "SIM115", # Use a context manager for opening files + "PLW1510", # `subprocess.run` without explicit `check` argument +] From fc797b3bf6f6dce134f01ef5af22428ab87ab6d0 Mon Sep 17 00:00:00 2001 From: Andrew Robbertz <24920994+Alrobbertz@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:57:38 -0400 Subject: [PATCH 2/2] Update AGENTS File --- AGENTS.md | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c993584 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,208 @@ +# AGENTS.md + +This file helps AI coding agents understand the repository structure, build/test conventions, and key architecture decisions for the **Space Weather SOC (SWSOC) AWS Lambda Executor Function**. + +## Project Overview + +This is an AWS Lambda function that implements an executor pattern to run scheduled tasks via CloudWatch Events/EventBridge rules. Each rule name maps directly to a corresponding executor function, enabling modular management of space weather data processing tasks. + +The function: +- Routes incoming CloudWatch/EventBridge events to appropriate handler functions +- Executes scheduled data collection and processing tasks +- Integrates with AWS Secrets Manager for credential management +- Handles multiple satellite/instrument data sources (GOES, STIX, PADRE, UDL REACH) +- Stores processed data in Amazon Timestream and S3 +- Creates Grafana annotations for solar events + +See [README.rst](README.rst) for detailed architecture and implementation details. + +## Essential Commands + +### Testing +```bash +# Run all tests with coverage +pytest --pyargs lambda_function/tests --cov=lambda_function/src --cov-report=html + +# Run a specific test file +pytest lambda_function/tests/test_executor.py -v + +# Run tests matching a pattern +pytest lambda_function/tests -k "import_goes" -v +``` + +### Linting & Code Style +```bash +# Check code with ruff +ruff check lambda_function/src lambda_function/tests + +# Format with ruff +ruff format lambda_function/src lambda_function/tests +``` + +### Docker & Local Lambda Testing +```bash +# Build Docker container for Lambda runtime +cd lambda_function && docker build -t sdc_aws_executor_lambda:latest . + +# Run Lambda locally and test with a sample event +docker run -p 9000:8080 sdc_aws_executor_lambda:latest + +# In another terminal, invoke the function with a test event +curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ + -d @lambda_function/tests/test_data/test_executor_event.json +``` + +### CodeBuild Deployment +```bash +# Build process defined in buildspec.yml +# Builds Docker image, pushes to ECR, prepares for Lambda deployment +``` + +## Project Structure + +``` +lambda_function/ +├── src/ +│ ├── lambda.py # Handler entry point for Lambda +│ └── executor/ +│ ├── __init__.py +│ ├── executor.py # Core Executor class with all functions +│ │ # - handle_event() - main dispatcher +│ │ # - import_GOES_data_to_timestream() +│ │ # - create_GOES_data_annotations() +│ │ # - import_STIX_to_timestream() +│ │ # - get_PADRE_orbit_data() +│ │ # - import_UDL_REACH_to_s3() +│ │ # - generate_loc_report() +│ └── config/ +│ └── ccsdspy/ +│ └── config.yml # CCSDSPY configuration for data parsing +├── tests/ +│ ├── conftest.py # pytest fixtures (fixtures, mocking setup) +│ ├── test_executor.py # Main test suite +│ └── test_data/ +│ └── test_executor_event.json # Sample EventBridge event +├── Dockerfile # Lambda container image (uses base from PADRE) +└── requirements.txt # Production dependencies + +Root project files: +├── buildspec.yml # AWS CodeBuild configuration +├── ruff.toml # Linting rules +├── requirements.dev.txt # Development dependencies (testing, linting) +└── README.rst # Full project documentation +``` + +## Key Technical Details + +**Python Version**: 3.9+ (Lambda runtime compatibility) + +**AWS Services**: +- **EventBridge/CloudWatch Events**: Triggers Lambda via rule names +- **Secrets Manager**: Stores credentials (Grafana API key, UDL auth, etc.) +- **Timestream**: Time-series database for satellite data +- **S3**: File storage for REACH and LOC report data +- **Lambda**: Function runtime environment + +**Environment Variables**: +- `SECRET_ARN_GRAFANA`: ARN for Grafana API credentials secret +- `SECRET_ARN_UDL`: ARN for UDL authentication credentials secret +- Fetched credentials are injected as environment variables in constructor + +**Core Dependencies** (from `requirements.txt`): +- `swxsoc`: SWxSOC core library (S3, Slack, logging, config utilities) +- `swxsoc_reach`: REACH data download utilities +- `padre_craft`: PADRE satellite orbit utilities +- `stixdcpy`: STIX X-ray data access +- `pandas`: Data manipulation +- AWS SDK (`boto3`): Built-in to Lambda + +**Development Dependencies** (from `requirements.dev.txt`): +- `pytest`, `pytest-astropy`, `pytest-cov`: Testing framework +- `moto==5.0.15`: AWS service mocking for tests +- `ruff`: Code linting and formatting + +**Linting Configuration** (see [ruff.toml](ruff.toml)): +- Ignores specific rules (EXE002, BLE001, TRY201, etc.) +- Applied to both `src/` and `tests/` directories + +## Testing Conventions + +- **Fixtures**: `conftest.py` provides: + - `isolate_executor_env`: Auto-applied fixture that clears credential-related env vars before each test + - `aws_credentials`: Provides fake AWS credentials for boto3 + - `mock_aws`: Mocks all AWS services with moto + +- **Mocking Strategy**: + - Use `moto` to mock AWS Secrets Manager, Timestream, S3, etc. + - Example: Create mock secrets in Secrets Manager and set `SECRET_ARN_*` env vars before instantiating `Executor` + - Patch external dependencies (e.g., `stixdcpy.LightCurves.from_sdc`) to avoid downloading large datasets during tests + +- **Test File Naming**: Place tests in `lambda_function/tests/test_*.py` +- **Running Subsets**: Use `-k` flag to run tests matching a pattern (e.g., `-k "import_"`) + +## CI/CD Workflows + +CI/CD is configured in [.github/workflows/](.github/workflows/): +- **testing.yml**: Runs pytest with coverage on PR and scheduled basis +- **codestyle.yml**: Runs ruff linting checks + +## Adding New Executor Functions + +1. Define a new method in the `Executor` class in [lambda_function/src/executor/executor.py](lambda_function/src/executor/executor.py) +2. The method name should match the EventBridge rule name (e.g., `my_new_function`) +3. Create a CloudWatch/EventBridge rule named `my_new_function` with desired schedule +4. Add that rule as a trigger to the executor Lambda function in AWS +5. Add unit tests in [lambda_function/tests/test_executor.py](lambda_function/tests/test_executor.py) +6. Document the function in [README.rst](README.rst) + +Key pattern for functions: +```python +@staticmethod # or instance method if state needed +def my_new_function() -> None: + """Brief description of what the function does.""" + # Implementation here + # Use swxsoc utilities for common operations +``` + +## Key Decisions & Patterns + +1. **Rule Name → Function Mapping**: The EventBridge rule name is extracted from the event and used to dispatch to a corresponding executor method. This decouples scheduling configuration from code. + +2. **Secrets Manager Integration**: Credentials are loaded in the `Executor` constructor and injected as environment variables. This avoids hardcoding secrets and supports multiple deployment environments. + +3. **Modular Data Handling**: All AWS service interactions (S3, Timestream, Secrets) and data source APIs are abstracted via utility libraries (`swxsoc`, `swxsoc_reach`, etc.), keeping executor functions focused on business logic. + +4. **Comprehensive Test Coverage**: Tests use `moto` to mock AWS services and avoid requiring actual AWS credentials or external data sources during CI/CD. + +5. **Docker for Local Testing**: The Dockerfile ensures the exact Lambda runtime environment can be tested locally before deployment. + +6. **Static Methods**: Most executor functions are static methods since they don't maintain state, simplifying testing and invocation. + +## Deployment + +The function is deployed via AWS CodeBuild using [buildspec.yml](buildspec.yml): +- Builds a Docker image with the Lambda function +- Pushes to private ECR (tagging by git tag or timestamp) +- Updates Lambda function configuration to use the new image + +## Common Development Tasks + +| Task | Command/Approach | +|------|------------------| +| Run all tests | `pytest --pyargs lambda_function/tests --cov=lambda_function/src --cov-report=html` | +| Check linting | `ruff check lambda_function/src lambda_function/tests` | +| Format code | `ruff format lambda_function/src lambda_function/tests` | +| Build Lambda image locally | `cd lambda_function && docker build -t sdc_aws_executor_lambda:latest .` | +| Test Lambda locally | See Docker commands above; use `test_executor_event.json` as sample | +| Add new executor function | Add method to `Executor` class, create EventBridge rule, add unit test | +| Update dependencies | Edit `lambda_function/requirements.txt` (prod) or `requirements.dev.txt` (dev); rebuild Docker image | +| View test coverage | After running pytest with `--cov-report=html`, open `htmlcov/index.html` | +| Run a single test | `pytest lambda_function/tests/test_executor.py::test_constructor_loads_secrets_from_moto -v` | + +## Questions or Issues? + +- For architecture questions, refer to [README.rst](README.rst#Architecture) +- For swxsoc library details, see the [swxsoc repository](https://github.com/swxsoc/swxsoc) +- For swxsoc_reach details, see the [swxsoc_reach repository](https://github.com/swxsoc/swxsoc_reach) +- For CI/CD configuration, check [.github/workflows/](.github/workflows/) +- For AWS Lambda deployment details, see [buildspec.yml](buildspec.yml)