Skip over bad entries in UAL while parsing data

dissect/database/ese/record.py

@@ -199,6 +199,27 @@ def __init__(self, table: Table, node: Node):
         self._get_tag_field = lru_cache(4096)(self._get_tag_field)
         self._find_tag_field_idx = lru_cache(4096)(self._find_tag_field_idx)
 
+    @property
+    def is_empty(self) -> bool:
+        """Return whether the record is a tombstone or empty (has no defined column data)."""
+        return self._last_fixed_id == 0 and self._last_variable_id == 0 and self._tagged_data_count == 0
+
+    def matches_schema(self) -> bool:
+        """Check if the record column layout is compatible with the table schema.
+
+        A record can have fewer columns than the table defines — missing columns are null.
+        A record with more columns most likely is corrupted (or written with another schema)
+        and cannot be parsed.
+        """
+        num_fixed, num_variable, num_tagged = self.table.column_counts
+        return all(
+            (
+                self._last_fixed_id <= num_fixed,
+                (self._last_variable_id - 127) <= num_variable,
+                self._tagged_data_count <= num_tagged,
+            )
+        )
+
     def get(self, column: Column, raw: bool = False, errors: str | None = "backslashreplace") -> RecordValue:
         """Retrieve the value for the specified column.
 
@@ -401,6 +422,8 @@ def _get_tagged(self, column: Column) -> bytes | None:
             if not tag_field.is_null:
                 offset = self._tagged_data_start
                 value = self.data[offset + data_start : offset + data_end]
+            else:
+                value = None
         else:
             # If the column has a default, use that
             # If not, this defaults to None

dissect/database/ese/table.py

@@ -115,6 +115,19 @@ def column_names(self) -> list[str]:
         """Return a list of all the column names."""
         return list(self._column_name_map.keys())
 
+    @cached_property
+    def column_counts(self) -> tuple[int, int, int]:
+        """Return the number of fixed, variable and tagged columns in this table.
+
+        Returns:
+            A tuple of (fixed, variable, tagged) column counts.
+        """
+        return (
+            sum(c.is_fixed for c in self.columns),
+            sum(c.is_variable for c in self.columns),
+            sum(c.is_tagged for c in self.columns),
+        )
+
     @property
     def primary_index(self) -> Index | None:
         # It's generally the first index, but loop just in case

dissect/database/ese/tools/ual.py

@@ -34,6 +34,7 @@ class UAL:
         "InsertDate",
         "LastAccess",
         "LastSeen",
+        "LastSeenActive",  # Present in VIRTUALMACHINES table
     )
 
     def __init__(self, fh: BinaryIO):
@@ -49,6 +50,9 @@ def get_table_records(self, table_name: str) -> Iterator[dict[str, UalValue]]:
             return None
 
         for record in table.records():
+            if record._data.is_empty or not record._data.matches_schema():
+                continue
+
             record_data = {}
 
             last_access_year = None

tests/ese/conftest.py

@@ -63,3 +63,8 @@ def ual_db() -> Iterator[BinaryIO]:
 @pytest.fixture
 def certlog_db() -> Iterator[BinaryIO]:
     yield from open_file_gz("_data/ese/tools/CertLog.edb.gz")
+
+
+@pytest.fixture
+def ual_bad_entries_db() -> Iterator[BinaryIO]:
+    yield from open_file_gz("_data/ese/tools/SumBadEntries.mdb.gz")

tests/ese/test_table.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+from typing import BinaryIO
 from unittest.mock import MagicMock
 
+from dissect.database.ese.ese import ESE
+from dissect.database.ese.record import RecordData
 from dissect.database.ese.table import Table
 
 
@@ -32,3 +35,29 @@ def test_find_index() -> None:
     assert table.find_index(["UnsignedByte"]) is None
     assert table.find_index(["Id", "Bit"]) == mock_idx_id
     assert table.find_index(["Bit", "SomethingElse"]) == mock_idx_bit
+
+
+def test_record_data_bad_entries(ual_bad_entries_db: BinaryIO) -> None:
+    db = ESE(ual_bad_entries_db)
+
+    clients = db.table("CLIENTS")
+    assert len(list(clients.records())) == 24
+
+    dns = db.table("DNS")
+    assert len(list(dns.records())) == 17
+
+    all_clients_nodes = list(clients.root.iter_leaf_nodes())
+    tombstones = sum(1 for n in all_clients_nodes if RecordData(clients, n).is_empty)
+    assert tombstones == 5
+
+    dns_tombstones = 0
+    dns_mismatch = 0
+    for node in dns.root.iter_leaf_nodes():
+        rd = RecordData(dns, node)
+        if rd.is_empty:
+            dns_tombstones += 1
+        elif not rd.matches_schema():
+            dns_mismatch += 1
+
+    assert dns_tombstones == 3
+    assert dns_mismatch == 2

tests/ese/tools/test_ual.py

@@ -13,3 +13,16 @@ def test_ual(ual_db: BinaryIO) -> None:
     assert len(list(db.get_table_records("VIRTUALMACHINES"))) == 0
     assert len(list(db.get_table_records("DNS"))) == 12
     assert len(list(db.get_table_records("SYSTEM_IDENTITY"))) == 0
+
+
+def test_ual_skip_bad_entries(ual_bad_entries_db: BinaryIO) -> None:
+    ual = UAL(ual_bad_entries_db)
+
+    # CLIENTS have 24 entries (5 are empty (tombstones))
+    # DNS have 17 (3 empty and 2 with schema mismatch)
+    # They should be skipped
+    clients = list(ual.get_table_records("CLIENTS"))
+    dns = list(ual.get_table_records("DNS"))
+
+    assert len(clients) == 19
+    assert len(dns) == 12