diff --git a/src/data/archive.py b/src/data/archive.py index 8e895f5..0839d03 100755 --- a/src/data/archive.py +++ b/src/data/archive.py @@ -162,15 +162,21 @@ def copy_to_M3(self, resampled_nc_file: str) -> None: pass def _archive_file(self, src_file: Path, dst_file: Path) -> None: - """Copy src to dst; skip if dst already exists unless --clobber.""" + """Copy src to dst; skip only if dst already exists and is at least as + new as src. --clobber always forces a copy regardless of freshness. + """ if not src_file.exists(): + self.logger.debug("Source file not found, skipping: %s", src_file) return if dst_file.exists(): if self.clobber: dst_file.unlink() - else: - self.logger.info("Already archived, skipping: %s", dst_file.name) + elif dst_file.stat().st_mtime >= src_file.stat().st_mtime: + self.logger.info("Already archived and up to date, skipping: %s", dst_file.name) return + else: + self.logger.info("Archived copy of %s is stale, re-archiving", dst_file.name) + dst_file.unlink() shutil.copyfile(src_file, dst_file) self.logger.info("copyfile %s %s done.", src_file.name, dst_file.parent) @@ -229,20 +235,7 @@ def copy_sbd_to_LRAUV(self, sbd_nc_path: Path) -> None: stem = sbd_nc_path.stem # e.g. ahi_20260317_20260318_sbd_1S for pattern in (f"{stem}.nc", f"{stem}_*.png", f"{stem}_*.txt"): for src_file in sorted(src_dir.glob(pattern)): - dst_file = dst_dir / src_file.name - if self.clobber: - if dst_file.exists(): - self.logger.info("Removing %s", dst_file) - dst_file.unlink() - self.logger.info("copyfile %s %s", src_file, dst_dir) - shutil.copyfile(src_file, dst_file) - self.logger.info("copyfile %s %s done.", src_file, dst_dir) - elif not dst_file.exists(): - self.logger.info("copyfile %s %s", src_file, dst_dir) - shutil.copyfile(src_file, dst_file) - self.logger.info("copyfile %s %s done.", src_file, dst_dir) - else: - self.logger.info("%s exists, not overwriting (use --clobber)", dst_file.name) + self._archive_file(src_file, dst_dir / src_file.name) def copy_lrauv_deployment(self, deployment_dir: Path, plot_name_stem: str) -> None: """Copy LRAUV deployment plots and HTML index to the LRAUV archive volume. @@ -285,23 +278,7 @@ def copy_lrauv_deployment(self, deployment_dir: Path, plot_name_stem: str) -> No if index_src.exists(): candidates.append((index_src, dst_dir.parent / index_src.name)) for src_file, dst_file in candidates: - if not src_file.exists(): - self.logger.debug("Source file not found, skipping: %s", src_file) - continue - if self.clobber: - if dst_file.exists(): - self.logger.info("Removing %s", dst_file) - dst_file.unlink() - shutil.copyfile(src_file, dst_file) - self.logger.info("copyfile %s %s done.", src_file.name, dst_file.parent) - elif dst_file.exists(): - self.logger.info( - "%-60s exists, but is not being archived because --clobber is not specified.", - src_file.name, - ) - else: - shutil.copyfile(src_file, dst_file) - self.logger.info("copyfile %s %s done.", src_file.name, dst_dir) + self._archive_file(src_file, dst_file) def process_command_line(self): """Process command line arguments using shared parser infrastructure.""" diff --git a/src/data/process.py b/src/data/process.py index d859d4d..5eef87c 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -139,6 +139,17 @@ class Processor: logger.addHandler(_handler) _log_levels = (logging.WARN, logging.INFO, logging.DEBUG) + # Maps auv_name to the entry-point script of each Processor subclass that + # goes through process_mission(), used only to label the provenance + # record's script_name there. LRAUV ("tethys" et al.) is deliberately + # excluded: it's processed via process_log_file() instead, which has its + # own separate, correctly-hardcoded script_name. + _AUV_SCRIPT_NAMES = { + "dorado": "src/data/process_dorado.py", + "i2map": "src/data/process_i2map.py", + "Dorado389": "src/data/process_Dorado389.py", + } + def __init__(self, auv_name, vehicle_dir, mount_dir, calibration_dir, config=None) -> None: # noqa: PLR0913 # Variables to be set by subclasses, e.g.: # auv_name = "i2map" @@ -1017,6 +1028,14 @@ def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C90 self.resample(mission) self.create_products(mission) if self.config["update_ssds_provenance"]: + script_name = self._AUV_SCRIPT_NAMES.get(self.auv_name) + if script_name is None: + self.logger.warning( + "No script_name mapping for auv_name %r;" + " provenance script_name will default to process.py", + self.auv_name, + ) + script_name = "src/data/process.py" self._submit_provenance( output_nc=str( Path( @@ -1039,7 +1058,7 @@ def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C90 ], pr_start=_pr_start, pr_end=datetime.now(tz=UTC).isoformat(), - script_name="src/data/process_dorado.py", + script_name=script_name, log_file=str( Path( self.config["base_path"], diff --git a/src/data/test_archive.py b/src/data/test_archive.py new file mode 100644 index 0000000..5f8fb46 --- /dev/null +++ b/src/data/test_archive.py @@ -0,0 +1,175 @@ +# noqa: INP001 +"""Tests for Archiver._archive_file() — freshness-aware archive copying.""" + +import os +from pathlib import Path +from unittest.mock import patch + +import pytest +from archive import Archiver + + +@pytest.fixture() +def arch(): + return Archiver(add_handlers=False, clobber=False) + + +def _touch_with_mtime(path: Path, content: str, mtime: float) -> None: + path.write_text(content) + os.utime(path, (mtime, mtime)) + + +class TestArchiveFile: + def test_copies_when_dst_missing(self, arch, tmp_path): + src = tmp_path / "src.nc" + dst = tmp_path / "dst.nc" + src.write_text("fresh data") + + arch._archive_file(src, dst) + + assert dst.exists() # noqa: S101 + assert dst.read_text() == "fresh data" # noqa: S101 + + def test_noop_when_src_missing(self, arch, tmp_path): + src = tmp_path / "ghost.nc" + dst = tmp_path / "dst.nc" + + arch._archive_file(src, dst) + + assert not dst.exists() # noqa: S101 + + def test_skips_when_dst_is_fresh(self, arch, tmp_path): + """dst newer than src (the common case: dst was archived after src was + last written) must be left alone — this is the normal "already + archived, nothing changed" case.""" + src = tmp_path / "src.nc" + dst = tmp_path / "dst.nc" + _touch_with_mtime(src, "old data", mtime=1000) + _touch_with_mtime(dst, "already archived", mtime=2000) + + arch._archive_file(src, dst) + + assert dst.read_text() == "already archived" # noqa: S101 + + def test_recopies_when_dst_is_stale(self, arch, tmp_path): + """dst older than src (mission was reprocessed after it was archived) + must be refreshed even without --clobber — this is the regression + this test guards against: a stale archived copy silently never being + updated.""" + src = tmp_path / "src.nc" + dst = tmp_path / "dst.nc" + _touch_with_mtime(src, "reprocessed data", mtime=2000) + _touch_with_mtime(dst, "stale archived data", mtime=1000) + + arch._archive_file(src, dst) + + assert dst.read_text() == "reprocessed data" # noqa: S101 + + def test_clobber_overwrites_even_when_dst_is_fresh(self, tmp_path): + """--clobber must force a copy regardless of freshness.""" + arch = Archiver(add_handlers=False, clobber=True) + src = tmp_path / "src.nc" + dst = tmp_path / "dst.nc" + _touch_with_mtime(src, "new data", mtime=1000) + _touch_with_mtime(dst, "old data but newer mtime", mtime=2000) + + arch._archive_file(src, dst) + + assert dst.read_text() == "new data" # noqa: S101 + + +class TestCopySbdToLRAUV: + """copy_sbd_to_LRAUV() now delegates to _archive_file() for every file, + so it must pick up freshness the same way copy_to_LRAUV() does.""" + + def test_recopies_stale_product_without_clobber(self, arch, tmp_path): + base_lrauv_path = tmp_path / "local" + lrauv_vol = tmp_path / "vol" + rel = Path("ahi/realtime/sbdlogs/2026/20260317_20260318") + src_dir = base_lrauv_path / rel + dst_dir = lrauv_vol / rel + src_dir.mkdir(parents=True) + dst_dir.mkdir(parents=True) + + stem = "ahi_20260317_20260318_sbd_1S" + _touch_with_mtime(src_dir / f"{stem}.nc", "reprocessed", mtime=2000) + _touch_with_mtime(dst_dir / f"{stem}.nc", "stale", mtime=1000) + + with ( + patch("archive.BASE_LRAUV_PATH", base_lrauv_path), + patch("archive.LRAUV_VOL", str(lrauv_vol)), + ): + arch.copy_sbd_to_LRAUV(src_dir / f"{stem}.nc") + + assert (dst_dir / f"{stem}.nc").read_text() == "reprocessed" # noqa: S101 + + def test_skips_fresh_product_without_clobber(self, arch, tmp_path): + base_lrauv_path = tmp_path / "local" + lrauv_vol = tmp_path / "vol" + rel = Path("ahi/realtime/sbdlogs/2026/20260317_20260318") + src_dir = base_lrauv_path / rel + dst_dir = lrauv_vol / rel + src_dir.mkdir(parents=True) + dst_dir.mkdir(parents=True) + + stem = "ahi_20260317_20260318_sbd_1S" + _touch_with_mtime(src_dir / f"{stem}.nc", "old", mtime=1000) + _touch_with_mtime(dst_dir / f"{stem}.nc", "already archived", mtime=2000) + + with ( + patch("archive.BASE_LRAUV_PATH", base_lrauv_path), + patch("archive.LRAUV_VOL", str(lrauv_vol)), + ): + arch.copy_sbd_to_LRAUV(src_dir / f"{stem}.nc") + + assert (dst_dir / f"{stem}.nc").read_text() == "already archived" # noqa: S101 + + +class TestCopyLrauvDeployment: + """copy_lrauv_deployment() now delegates to _archive_file() for every file.""" + + def test_recopies_stale_index_without_clobber(self, arch, tmp_path): + base_lrauv_path = tmp_path / "local" + lrauv_vol = tmp_path / "vol" + rel = Path("ahi/missionlogs/2026/20260317_20260318") + deployment_dir = base_lrauv_path / rel + dst_dir = lrauv_vol / rel + deployment_dir.mkdir(parents=True) + dst_dir.mkdir(parents=True) + + stem = "CANON_March_2026" + _touch_with_mtime(deployment_dir / f"{stem}_2column_cmocean.png", "reprocessed", mtime=2000) + _touch_with_mtime(dst_dir / f"{stem}_2column_cmocean.png", "stale", mtime=1000) + + with ( + patch("archive.BASE_LRAUV_PATH", base_lrauv_path), + patch("archive.LRAUV_VOL", str(lrauv_vol)), + ): + arch.copy_lrauv_deployment(deployment_dir, stem) + + assert ( # noqa: S101 + dst_dir / f"{stem}_2column_cmocean.png" + ).read_text() == "reprocessed" + + def test_skips_fresh_index_without_clobber(self, arch, tmp_path): + base_lrauv_path = tmp_path / "local" + lrauv_vol = tmp_path / "vol" + rel = Path("ahi/missionlogs/2026/20260317_20260318") + deployment_dir = base_lrauv_path / rel + dst_dir = lrauv_vol / rel + deployment_dir.mkdir(parents=True) + dst_dir.mkdir(parents=True) + + stem = "CANON_March_2026" + _touch_with_mtime(deployment_dir / f"{stem}_2column_cmocean.png", "old", mtime=1000) + _touch_with_mtime(dst_dir / f"{stem}_2column_cmocean.png", "already archived", mtime=2000) + + with ( + patch("archive.BASE_LRAUV_PATH", base_lrauv_path), + patch("archive.LRAUV_VOL", str(lrauv_vol)), + ): + arch.copy_lrauv_deployment(deployment_dir, stem) + + assert ( # noqa: S101 + dst_dir / f"{stem}_2column_cmocean.png" + ).read_text() == "already archived"