3274 hi goodtimes filter out gain tests

imap_processing/cdf/config/imap_hi_variable_attrs.yaml

@@ -446,8 +446,10 @@ hi_de_ccsds_qf:
   VALIDMIN: 0
   VAR_NOTES: >
     Bitwise quality flag for CCSDS packet. Bit 0 (value 1): packet was full
-    (contained 664 events). Bit 2 (value 4): packet contained events from an
-    invalid spin.
+    (contained 664 events). Bit 1 (value 2): ESA (inner/outer) voltage did not
+    match any known esa_energy_step. Bit 2 (value 4): packet contained events
+    from an invalid spin. Bit 3 (value 8): detector high voltage (CEM/MCP/U-Can/
+    deflector) did not match the pointing's classified gain configuration.
 
 # ======= L1C PSET Section =======
 
@@ -477,6 +479,24 @@ hi_pset_esa_energy_step:
   VAR_TYPE: support_data
   dtype: uint8
 
+hi_pset_geometric_factor:
+  <<: *default_float32
+  CATDESC: Geometric factor for the detector gain configuration active during this pointing
+  DEPEND_0: epoch
+  DEPEND_1: esa_energy_step
+  DISPLAY_TYPE: no_plot
+  FIELDNAM: Geometric factor
+  FORMAT: E12.5
+  LABLAXIS: Geometric Factor
+  LABL_PTR_1: esa_energy_step_label
+  UNITS: cm^2 sr
+  VALIDMAX: 1.0
+  VALIDMIN: 0.0
+  VAR_NOTES: >
+    Geometric factor per esa_energy_step, looked up from the gain-configuration
+    ancillary file using the l1b_de's classified gain configuration.
+  VAR_TYPE: support_data
+
 hi_pset_calibration_prod:
   CATDESC: Coincidence types are combined into various calibration products
   FIELDNAM: Calibration product number

imap_processing/cli.py

@@ -941,9 +941,17 @@ def do_processing(  # noqa: PLR0912
                 l1b_hk_file = dependencies.get_file_paths(
                     source="hi", data_type="l1b", descriptor="hk"
                 )[0]
-                esa_energies_csv = dependencies.get_file_paths(data_type="ancillary")[0]
+                esa_energies_csv = dependencies.get_file_paths(
+                    data_type="ancillary", descriptor="esa-energies"
+                )[0]
+                gain_config_csv = dependencies.get_file_paths(
+                    data_type="ancillary", descriptor="gain-configuration"
+                )[0]
                 datasets = hi_l1b.annotate_direct_events(
-                    load_cdf(l1a_de_file), load_cdf(l1b_hk_file), esa_energies_csv
+                    load_cdf(l1a_de_file),
+                    load_cdf(l1b_hk_file),
+                    esa_energies_csv,
+                    gain_config_csv,
                 )
         elif self.data_level == "l1c":
             if "pset" in self.descriptor:
@@ -961,10 +969,10 @@ def do_processing(  # noqa: PLR0912
                 anc_dependencies = dependencies.get_processing_inputs(
                     data_type="ancillary"
                 )
-                if len(anc_dependencies) != 2:
+                if len(anc_dependencies) != 3:
                     raise ValueError(
-                        f"Expected two ancillary dependencies (cal-prod and "
-                        f"backgrounds). Got "
+                        f"Expected three ancillary dependencies (cal-prod, "
+                        f"backgrounds, and gain-configuration). Got "
                         f"{[anc_dep.descriptor for anc_dep in anc_dependencies]}"
                     )
 
@@ -976,14 +984,16 @@ def do_processing(  # noqa: PLR0912
                     for dep in anc_dependencies
                 }
 
-                # Verify we have both required ancillary files
+                # Verify we have all required ancillary files
                 if (
                     "cal-prod" not in anc_path_dict
                     or "backgrounds" not in anc_path_dict
+                    or "gain-configuration" not in anc_path_dict
                 ):
                     raise ValueError(
-                        f"Missing required ancillary files. Expected 'cal-prod' and "
-                        f"'backgrounds', got {list(anc_path_dict.keys())}"
+                        f"Missing required ancillary files. Expected 'cal-prod', "
+                        f"'backgrounds', and 'gain-configuration', got "
+                        f"{list(anc_path_dict.keys())}"
                     )
 
                 # Load goodtimes dependency
@@ -1001,6 +1011,7 @@ def do_processing(  # noqa: PLR0912
                     anc_path_dict["cal-prod"],
                     load_cdf(goodtimes_paths[0]),
                     anc_path_dict["backgrounds"],
+                    anc_path_dict["gain-configuration"],
                 )
         elif self.data_level == "l2":
             science_paths = dependencies.get_file_paths(source="hi", data_type="l1c")

imap_processing/hi/hi_goodtimes.py

@@ -53,7 +53,7 @@ class CullCode(IntEnum):
     STAT_FILTER_0 = 1 << 4  # 16
     STAT_FILTER_1 = 1 << 5  # 32
     STAT_FILTER_2 = 1 << 6  # 64
-    BAD_ESA_VOLTAGE = 1 << 7  # 128
+    BAD_HV_VALUE = 1 << 7  # 128
 
 
 def hi_goodtimes(
@@ -69,7 +69,8 @@ def hi_goodtimes(
     This is the top-level function that orchestrates all goodtimes culling
     operations for a single pointing. It applies the following filters in order:
 
-    0. mark_bad_esa_voltage - Remove times with invalid ESA voltage configuration
+    0. mark_bad_voltage - Remove times with invalid ESA or detector HV
+       configuration (e.g. gain test intervals)
     1. mark_incomplete_spin_sets - Remove incomplete 8-spin histogram periods
     2. mark_drf_times - Remove times during spacecraft drift restabilization
     3. mark_bad_tdc_cal - Remove times with failed TDC calibration
@@ -276,9 +277,9 @@ def _apply_goodtimes_filters(
 
     # === Apply culling filters ===
 
-    # 0. Mark bad ESA voltage times
-    logger.info("Applying filter: mark_bad_esa_voltage")
-    mark_bad_esa_voltage(goodtimes_ds, current_l1b_de)
+    # 0. Mark bad ESA/detector HV voltage times
+    logger.info("Applying filter: mark_bad_voltage")
+    mark_bad_voltage(goodtimes_ds, current_l1b_de)
 
     # 1. Mark incomplete spin sets
     logger.info("Applying filter: mark_incomplete_spin_sets")
@@ -953,22 +954,28 @@ def finalize_dataset(self) -> xr.Dataset:
 # ==============================================================================
 
 
-def mark_bad_esa_voltage(
+def mark_bad_voltage(
     goodtimes_ds: xr.Dataset,
     l1b_de: xr.Dataset,
-    cull_code: int = CullCode.BAD_ESA_VOLTAGE,
+    cull_code: int = CullCode.BAD_HV_VALUE,
 ) -> None:
     """
-    Mark times when ESA voltages don't match expected values.
+    Mark times when ESA or detector high voltages don't match expected values.
 
     Filters out 8-spin periods where the ESA energy step is invalid, indicating
-    either calibration mode (esa_energy_step=0) or an ESA voltage mismatch
+    either calibration mode (esa_energy_step=0) or a voltage mismatch
     (esa_energy_step=FILLVAL). The voltage validation is performed during L1B
-    processing by matching measured inner/outer ESA voltages against the ESA
-    energies lookup table.
+    processing (see hi_l1b.get_esa_to_esa_energy_step_lut()) by matching
+    measured inner/outer ESA voltages against the ESA energies lookup table,
+    and matching measured detector high voltages (CEM/MCP/U-Can/deflector)
+    against the pointing's classified gain configuration. Both checks set
+    esa_energy_step=FILLVAL on mismatch; see the L1B DE ccsds_qf quality flag
+    (ImapHiL1bDeFlags.BAD_ESA_VOLTAGE / BAD_DETECTOR_VOLTAGE) to distinguish
+    which check failed.
 
     Algorithm Document Reference:
-        Section 2.3.2: Good times selection requiring valid ESA configuration
+        Section 2.2.4/2.3.2: Good times selection requiring valid ESA and
+        detector high voltage configuration.
 
     Parameters
     ----------
@@ -977,7 +984,7 @@ def mark_bad_esa_voltage(
     l1b_de : xarray.Dataset
         L1B Direct Event data containing esa_energy_step field.
     cull_code : int, optional
-        Cull code to use for marking bad times (default: CullCode.BAD_ESA_VOLTAGE).
+        Cull code to use for marking bad times (default: CullCode.BAD_HV_VALUE).
 
     Notes
     -----
@@ -986,10 +993,11 @@ def mark_bad_esa_voltage(
 
     Invalid ESA energy steps:
     - esa_energy_step = 0: Calibration mode (ESA stepping but not science data)
-    - esa_energy_step = FILLVAL (255): ESA voltage mismatch - measured voltages
-      didn't match any known energy step in the lookup table
+    - esa_energy_step = FILLVAL (255): ESA or detector voltage mismatch -
+      measured voltages didn't match any known energy step / gain
+      configuration
     """
-    logger.info("Running mark_bad_esa_voltage culling")
+    logger.info("Running mark_bad_voltage culling")
 
     # Get FILLVAL from attributes (should be 255 for uint8)
     fillval = l1b_de["esa_energy_step"].attrs.get("FILLVAL", 255)