diff --git a/dissect/database/ese/record.py b/dissect/database/ese/record.py index 639cafd..189cba1 100644 --- a/dissect/database/ese/record.py +++ b/dissect/database/ese/record.py @@ -199,6 +199,27 @@ def __init__(self, table: Table, node: Node): self._get_tag_field = lru_cache(4096)(self._get_tag_field) self._find_tag_field_idx = lru_cache(4096)(self._find_tag_field_idx) + @property + def is_empty(self) -> bool: + """Return whether the record is a tombstone or empty (has no defined column data).""" + return self._last_fixed_id == 0 and self._last_variable_id == 0 and self._tagged_data_count == 0 + + def matches_schema(self) -> bool: + """Check if the record column layout is compatible with the table schema. + + A record can have fewer columns than the table defines — missing columns are null. + A record with more columns most likely is corrupted (or written with another schema) + and cannot be parsed. + """ + num_fixed, num_variable, num_tagged = self.table.column_counts + return all( + ( + self._last_fixed_id <= num_fixed, + (self._last_variable_id - 127) <= num_variable, + self._tagged_data_count <= num_tagged, + ) + ) + def get(self, column: Column, raw: bool = False, errors: str | None = "backslashreplace") -> RecordValue: """Retrieve the value for the specified column. @@ -401,6 +422,8 @@ def _get_tagged(self, column: Column) -> bytes | None: if not tag_field.is_null: offset = self._tagged_data_start value = self.data[offset + data_start : offset + data_end] + else: + value = None else: # If the column has a default, use that # If not, this defaults to None diff --git a/dissect/database/ese/table.py b/dissect/database/ese/table.py index bc8a7b1..cc0d101 100644 --- a/dissect/database/ese/table.py +++ b/dissect/database/ese/table.py @@ -115,6 +115,19 @@ def column_names(self) -> list[str]: """Return a list of all the column names.""" return list(self._column_name_map.keys()) + @cached_property + def column_counts(self) -> tuple[int, int, int]: + """Return the number of fixed, variable and tagged columns in this table. + + Returns: + A tuple of (fixed, variable, tagged) column counts. + """ + return ( + sum(c.is_fixed for c in self.columns), + sum(c.is_variable for c in self.columns), + sum(c.is_tagged for c in self.columns), + ) + @property def primary_index(self) -> Index | None: # It's generally the first index, but loop just in case diff --git a/dissect/database/ese/tools/ual.py b/dissect/database/ese/tools/ual.py index 37f7e3a..126b122 100644 --- a/dissect/database/ese/tools/ual.py +++ b/dissect/database/ese/tools/ual.py @@ -34,6 +34,7 @@ class UAL: "InsertDate", "LastAccess", "LastSeen", + "LastSeenActive", # Present in VIRTUALMACHINES table ) def __init__(self, fh: BinaryIO): @@ -49,6 +50,9 @@ def get_table_records(self, table_name: str) -> Iterator[dict[str, UalValue]]: return None for record in table.records(): + if record._data.is_empty or not record._data.matches_schema(): + continue + record_data = {} last_access_year = None diff --git a/tests/_data/ese/tools/SumBadEntries.mdb.gz b/tests/_data/ese/tools/SumBadEntries.mdb.gz new file mode 100644 index 0000000..98aa688 --- /dev/null +++ b/tests/_data/ese/tools/SumBadEntries.mdb.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e68cd66484512140c627d55e33b00f50092902d1a47f97806214c86d067f110 +size 58638 diff --git a/tests/ese/conftest.py b/tests/ese/conftest.py index 4e9f4f1..a920d3d 100644 --- a/tests/ese/conftest.py +++ b/tests/ese/conftest.py @@ -63,3 +63,8 @@ def ual_db() -> Iterator[BinaryIO]: @pytest.fixture def certlog_db() -> Iterator[BinaryIO]: yield from open_file_gz("_data/ese/tools/CertLog.edb.gz") + + +@pytest.fixture +def ual_bad_entries_db() -> Iterator[BinaryIO]: + yield from open_file_gz("_data/ese/tools/SumBadEntries.mdb.gz") diff --git a/tests/ese/test_table.py b/tests/ese/test_table.py index 621e5f8..368a160 100644 --- a/tests/ese/test_table.py +++ b/tests/ese/test_table.py @@ -1,7 +1,10 @@ from __future__ import annotations +from typing import BinaryIO from unittest.mock import MagicMock +from dissect.database.ese.ese import ESE +from dissect.database.ese.record import RecordData from dissect.database.ese.table import Table @@ -32,3 +35,29 @@ def test_find_index() -> None: assert table.find_index(["UnsignedByte"]) is None assert table.find_index(["Id", "Bit"]) == mock_idx_id assert table.find_index(["Bit", "SomethingElse"]) == mock_idx_bit + + +def test_record_data_bad_entries(ual_bad_entries_db: BinaryIO) -> None: + db = ESE(ual_bad_entries_db) + + clients = db.table("CLIENTS") + assert len(list(clients.records())) == 24 + + dns = db.table("DNS") + assert len(list(dns.records())) == 17 + + all_clients_nodes = list(clients.root.iter_leaf_nodes()) + tombstones = sum(1 for n in all_clients_nodes if RecordData(clients, n).is_empty) + assert tombstones == 5 + + dns_tombstones = 0 + dns_mismatch = 0 + for node in dns.root.iter_leaf_nodes(): + rd = RecordData(dns, node) + if rd.is_empty: + dns_tombstones += 1 + elif not rd.matches_schema(): + dns_mismatch += 1 + + assert dns_tombstones == 3 + assert dns_mismatch == 2 diff --git a/tests/ese/tools/test_ual.py b/tests/ese/tools/test_ual.py index a72c542..1c94e97 100644 --- a/tests/ese/tools/test_ual.py +++ b/tests/ese/tools/test_ual.py @@ -13,3 +13,16 @@ def test_ual(ual_db: BinaryIO) -> None: assert len(list(db.get_table_records("VIRTUALMACHINES"))) == 0 assert len(list(db.get_table_records("DNS"))) == 12 assert len(list(db.get_table_records("SYSTEM_IDENTITY"))) == 0 + + +def test_ual_skip_bad_entries(ual_bad_entries_db: BinaryIO) -> None: + ual = UAL(ual_bad_entries_db) + + # CLIENTS have 24 entries (5 are empty (tombstones)) + # DNS have 17 (3 empty and 2 with schema mismatch) + # They should be skipped + clients = list(ual.get_table_records("CLIENTS")) + dns = list(ual.get_table_records("DNS")) + + assert len(clients) == 19 + assert len(dns) == 12