From 49cac8788dcb29933efe6328aebe49810773caa6 Mon Sep 17 00:00:00 2001 From: Stanislav Golovanov Date: Wed, 10 Jun 2026 18:09:05 +0300 Subject: [PATCH] Skip over bad entries in UAL while parsing data Some entries in UAL ese database can contain bad data - tombstoned records without data (no defined columns), or have mismatch between UAL table schema and actual number of columns in entry. Such records previously caused exceptions while parsing them, so instead parsing now we skip them. --- dissect/database/ese/record.py | 23 +++++++++++++++++ dissect/database/ese/table.py | 13 ++++++++++ dissect/database/ese/tools/ual.py | 4 +++ tests/_data/ese/tools/SumBadEntries.mdb.gz | 3 +++ tests/ese/conftest.py | 5 ++++ tests/ese/test_table.py | 29 ++++++++++++++++++++++ tests/ese/tools/test_ual.py | 13 ++++++++++ 7 files changed, 90 insertions(+) create mode 100644 tests/_data/ese/tools/SumBadEntries.mdb.gz diff --git a/dissect/database/ese/record.py b/dissect/database/ese/record.py index 639cafd..189cba1 100644 --- a/dissect/database/ese/record.py +++ b/dissect/database/ese/record.py @@ -199,6 +199,27 @@ def __init__(self, table: Table, node: Node): self._get_tag_field = lru_cache(4096)(self._get_tag_field) self._find_tag_field_idx = lru_cache(4096)(self._find_tag_field_idx) + @property + def is_empty(self) -> bool: + """Return whether the record is a tombstone or empty (has no defined column data).""" + return self._last_fixed_id == 0 and self._last_variable_id == 0 and self._tagged_data_count == 0 + + def matches_schema(self) -> bool: + """Check if the record column layout is compatible with the table schema. + + A record can have fewer columns than the table defines — missing columns are null. + A record with more columns most likely is corrupted (or written with another schema) + and cannot be parsed. + """ + num_fixed, num_variable, num_tagged = self.table.column_counts + return all( + ( + self._last_fixed_id <= num_fixed, + (self._last_variable_id - 127) <= num_variable, + self._tagged_data_count <= num_tagged, + ) + ) + def get(self, column: Column, raw: bool = False, errors: str | None = "backslashreplace") -> RecordValue: """Retrieve the value for the specified column. @@ -401,6 +422,8 @@ def _get_tagged(self, column: Column) -> bytes | None: if not tag_field.is_null: offset = self._tagged_data_start value = self.data[offset + data_start : offset + data_end] + else: + value = None else: # If the column has a default, use that # If not, this defaults to None diff --git a/dissect/database/ese/table.py b/dissect/database/ese/table.py index bc8a7b1..cc0d101 100644 --- a/dissect/database/ese/table.py +++ b/dissect/database/ese/table.py @@ -115,6 +115,19 @@ def column_names(self) -> list[str]: """Return a list of all the column names.""" return list(self._column_name_map.keys()) + @cached_property + def column_counts(self) -> tuple[int, int, int]: + """Return the number of fixed, variable and tagged columns in this table. + + Returns: + A tuple of (fixed, variable, tagged) column counts. + """ + return ( + sum(c.is_fixed for c in self.columns), + sum(c.is_variable for c in self.columns), + sum(c.is_tagged for c in self.columns), + ) + @property def primary_index(self) -> Index | None: # It's generally the first index, but loop just in case diff --git a/dissect/database/ese/tools/ual.py b/dissect/database/ese/tools/ual.py index 37f7e3a..126b122 100644 --- a/dissect/database/ese/tools/ual.py +++ b/dissect/database/ese/tools/ual.py @@ -34,6 +34,7 @@ class UAL: "InsertDate", "LastAccess", "LastSeen", + "LastSeenActive", # Present in VIRTUALMACHINES table ) def __init__(self, fh: BinaryIO): @@ -49,6 +50,9 @@ def get_table_records(self, table_name: str) -> Iterator[dict[str, UalValue]]: return None for record in table.records(): + if record._data.is_empty or not record._data.matches_schema(): + continue + record_data = {} last_access_year = None diff --git a/tests/_data/ese/tools/SumBadEntries.mdb.gz b/tests/_data/ese/tools/SumBadEntries.mdb.gz new file mode 100644 index 0000000..98aa688 --- /dev/null +++ b/tests/_data/ese/tools/SumBadEntries.mdb.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e68cd66484512140c627d55e33b00f50092902d1a47f97806214c86d067f110 +size 58638 diff --git a/tests/ese/conftest.py b/tests/ese/conftest.py index 4e9f4f1..a920d3d 100644 --- a/tests/ese/conftest.py +++ b/tests/ese/conftest.py @@ -63,3 +63,8 @@ def ual_db() -> Iterator[BinaryIO]: @pytest.fixture def certlog_db() -> Iterator[BinaryIO]: yield from open_file_gz("_data/ese/tools/CertLog.edb.gz") + + +@pytest.fixture +def ual_bad_entries_db() -> Iterator[BinaryIO]: + yield from open_file_gz("_data/ese/tools/SumBadEntries.mdb.gz") diff --git a/tests/ese/test_table.py b/tests/ese/test_table.py index 621e5f8..368a160 100644 --- a/tests/ese/test_table.py +++ b/tests/ese/test_table.py @@ -1,7 +1,10 @@ from __future__ import annotations +from typing import BinaryIO from unittest.mock import MagicMock +from dissect.database.ese.ese import ESE +from dissect.database.ese.record import RecordData from dissect.database.ese.table import Table @@ -32,3 +35,29 @@ def test_find_index() -> None: assert table.find_index(["UnsignedByte"]) is None assert table.find_index(["Id", "Bit"]) == mock_idx_id assert table.find_index(["Bit", "SomethingElse"]) == mock_idx_bit + + +def test_record_data_bad_entries(ual_bad_entries_db: BinaryIO) -> None: + db = ESE(ual_bad_entries_db) + + clients = db.table("CLIENTS") + assert len(list(clients.records())) == 24 + + dns = db.table("DNS") + assert len(list(dns.records())) == 17 + + all_clients_nodes = list(clients.root.iter_leaf_nodes()) + tombstones = sum(1 for n in all_clients_nodes if RecordData(clients, n).is_empty) + assert tombstones == 5 + + dns_tombstones = 0 + dns_mismatch = 0 + for node in dns.root.iter_leaf_nodes(): + rd = RecordData(dns, node) + if rd.is_empty: + dns_tombstones += 1 + elif not rd.matches_schema(): + dns_mismatch += 1 + + assert dns_tombstones == 3 + assert dns_mismatch == 2 diff --git a/tests/ese/tools/test_ual.py b/tests/ese/tools/test_ual.py index a72c542..1c94e97 100644 --- a/tests/ese/tools/test_ual.py +++ b/tests/ese/tools/test_ual.py @@ -13,3 +13,16 @@ def test_ual(ual_db: BinaryIO) -> None: assert len(list(db.get_table_records("VIRTUALMACHINES"))) == 0 assert len(list(db.get_table_records("DNS"))) == 12 assert len(list(db.get_table_records("SYSTEM_IDENTITY"))) == 0 + + +def test_ual_skip_bad_entries(ual_bad_entries_db: BinaryIO) -> None: + ual = UAL(ual_bad_entries_db) + + # CLIENTS have 24 entries (5 are empty (tombstones)) + # DNS have 17 (3 empty and 2 with schema mismatch) + # They should be skipped + clients = list(ual.get_table_records("CLIENTS")) + dns = list(ual.get_table_records("DNS")) + + assert len(clients) == 19 + assert len(dns) == 12