fix(ingestion): persist debris records and repair the model schema (fixes #117)

backend/models/db_models.py

@@ -418,7 +418,7 @@ class SpaceWeather(Base):
     k_index: Mapped[Optional[int]] = mapped_column(Integer)
     description: Mapped[Optional[str]] = mapped_column(Text)
     recorded_at: Mapped[datetime.datetime] = mapped_column(
-        DateTime(timezone=True), server_default=func.now(), index=True
+        DateTime(timezone=True), server_default=func.now()
     )
 
     @property

backend/orbital/spacetrack.py

@@ -199,7 +199,9 @@ def _gp_to_doc(
         epoch        = _parse_epoch(rec.get("EPOCH", ""))
         tle1         = rec.get("TLE_LINE1") or rec.get("LINE1")
         tle2         = rec.get("TLE_LINE2") or rec.get("LINE2")
-        now          = datetime.datetime.utcnow().isoformat()
+        # A real, timezone-aware datetime: `updatedAt` is a DateTime(timezone=True)
+        # column, not the ISO string the MongoDB schema used to store.
+        now          = datetime.datetime.now(datetime.timezone.utc)
 
         return {
             "noradId":          norad_id,
@@ -237,6 +239,14 @@ def _bulk_upsert(
             return 0, []
 
         model = Debris if is_debris else Satellite
+
+        # `_gp_to_doc` emits one satellite-shaped dict for every provider, but `debris` is
+        # a narrower table (no objectType/status/updatedAt/…). Passing those through made
+        # every debris write fail — CompileError on PostgreSQL, TypeError on SQLite — so
+        # project each doc onto the columns the target model actually has.
+        columns = {c.name for c in model.__table__.columns}
+        docs = [{k: v for k, v in doc.items() if k in columns} for doc in docs]
+
         failed: List[str] = []
         written = 0
 

backend/tests/test_ingestion.py

@@ -144,6 +144,48 @@ def test_rerun_does_not_create_duplicates(sqlite_db):
     assert sqlite_db.query(Satellite).count() == 1
 
 
+def test_debris_records_are_persisted(sqlite_db):
+    """
+    Regression: debris ingestion wrote nothing at all.
+
+    `_gp_to_doc` emits one satellite-shaped dict for every provider, but `debris` is a
+    narrower table. Passing the extra keys through made every debris write fail —
+    CompileError ("Unconsumed column names") on PostgreSQL, TypeError on SQLite — so the
+    catalog silently gained no debris on either backend.
+    """
+    from models.db_models import Debris
+
+    records = [
+        _make_gp_record("50032", "COSMOS 1408 DEB", "DEBRIS"),
+        _make_gp_record("50033", "FENGYUN 1C DEB", "DEBRIS"),
+    ]
+    svc = _build_service({"analyst": records})
+
+    status = svc.sync_group(sqlite_db, "analyst", "DEBRIS", limit=500)
+
+    assert status["upserted"] == 2
+    assert status["failed"] == 0
+    assert sqlite_db.query(Debris).count() == 2
+
+    row = sqlite_db.query(Debris).filter(Debris.noradId == "50032").one()
+    assert row.objectName == "COSMOS 1408 DEB"
+
+
+def test_satellite_timestamps_are_datetimes_not_strings(sqlite_db):
+    """
+    Regression: `updatedAt` was written as an ISO *string* (a MongoDB-era leftover) into a
+    DateTime(timezone=True) column, which SQLAlchemy's SQLite DateTime rejects outright.
+    """
+    import datetime
+    from models.db_models import Satellite
+
+    svc = _build_service({"active": [_make_gp_record("25544", "ISS")]})
+    svc.sync_group(sqlite_db, "active", "PAYLOAD", limit=500)
+
+    sat = sqlite_db.query(Satellite).one()
+    assert isinstance(sat.updatedAt, datetime.datetime)
+
+
 def test_all_groups_sync_returns_meaningful_status(sqlite_db):
     from models.db_models import Satellite, Debris
     payloads = {

backend/tests/test_models_schema.py

@@ -0,0 +1,65 @@
+"""
+Schema-level invariants for the SQLAlchemy models.
+
+These guard the metadata itself rather than any query, so a bad declaration is caught at
+collection time instead of surfacing as a confusing failure in an unrelated test.
+"""
+
+from collections import defaultdict
+
+import pytest
+from sqlalchemy import create_engine
+
+from database.session import Base
+import models.db_models  # noqa: F401 — registers all tables
+import orbital.providers.cache  # noqa: F401 — registers ProviderCache
+
+
+def test_index_names_are_unique_case_insensitively():
+    """
+    Regression: `spaceWeather.recorded_at` was indexed twice — once explicitly in
+    `__table_args__` as `ix_spaceweather_recorded_at`, and once implicitly via
+    `index=True`, which SQLAlchemy names `ix_spaceWeather_recorded_at`.
+
+    The two names differ only by case. SQLite treats identifiers case-insensitively, so
+    `create_all()` died with "index ix_spaceweather_recorded_at already exists" and took
+    every test that builds a schema down with it. PostgreSQL quotes the mixed-case name
+    and happily creates *two* redundant indexes on the same column instead.
+    """
+    collisions = []
+    for table in Base.metadata.sorted_tables:
+        by_lower = defaultdict(list)
+        for index in table.indexes:
+            by_lower[index.name.lower()].append(index.name)
+        collisions += [
+            f"{table.name}: {sorted(names)}"
+            for names in by_lower.values()
+            if len(names) > 1
+        ]
+
+    assert not collisions, f"Index names collide case-insensitively: {collisions}"
+
+
+def test_no_duplicate_indexes_on_the_same_columns():
+    """Two indexes over identical columns are dead weight on every write."""
+    duplicates = []
+    for table in Base.metadata.sorted_tables:
+        by_columns = defaultdict(list)
+        for index in table.indexes:
+            by_columns[tuple(c.name for c in index.columns)].append(index.name)
+        duplicates += [
+            f"{table.name}{cols}: {sorted(names)}"
+            for cols, names in by_columns.items()
+            if len(names) > 1
+        ]
+
+    assert not duplicates, f"Redundant indexes: {duplicates}"
+
+
+def test_create_all_succeeds_on_a_clean_database():
+    """The whole schema must build from scratch — this is what CI and a fresh dev setup do."""
+    engine = create_engine("sqlite://")
+    try:
+        Base.metadata.create_all(engine)
+    finally:
+        engine.dispose()