From 0e3eeb6eb417f4307276d327a5e3b59be32e404b Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Mon, 2 Feb 2026 16:05:56 +0100 Subject: [PATCH 01/19] Expand WAL validation --- dissect/database/sqlite3/sqlite3.py | 4 +- dissect/database/sqlite3/wal.py | 93 +++++++++++++++++++++- tests/sqlite3/test_wal.py | 118 +++++++++++++++++++++++++++- 3 files changed, 209 insertions(+), 6 deletions(-) diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py index e009053..ca19856 100644 --- a/dissect/database/sqlite3/sqlite3.py +++ b/dissect/database/sqlite3/sqlite3.py @@ -79,6 +79,7 @@ def __init__( fh: Path | BinaryIO, wal: WAL | Path | BinaryIO | None = None, checkpoint: Checkpoint | int | None = None, + validate_checksum: bool = False, ): if isinstance(fh, Path): path = fh @@ -90,6 +91,7 @@ def __init__( self.path = path self.wal = None self.checkpoint = None + self.validate_checksum = validate_checksum self.header = c_sqlite3.header(self.fh) if self.header.magic != SQLITE3_HEADER_MAGIC: @@ -211,7 +213,7 @@ def raw_page(self, num: int) -> bytes: # Check if the latest valid instance of the page is committed (either the frame itself # is the commit frame or it is included in a commit's frames). If so, return that frame's data. for commit in reversed(self.wal.commits): - if (frame := commit.get(num)) and frame.valid: + if (frame := commit.get(num)) and frame.valid(validate_checksum=self.validate_checksum): data = frame.data break diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 7d4ec76..028ddb2 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -125,13 +125,92 @@ def __init__(self, wal: WAL, offset: int): def __repr__(self) -> str: return f"" - @property - def valid(self) -> bool: + def valid(self, validate_checksum: bool = True) -> bool: + """Check if the frame is valid by comparing its salt values and optionally verifying the checksum. + + A frame is valid if: + - Its salt1 and salt2 values match those in the WAL header. + - Its checksum matches the calculated checksum. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + """ + return self.validate_salt() and self.validate_checksum() if validate_checksum else self.validate_salt() + + def validate_salt(self) -> bool: + """Check if the frame's salt values match those in the WAL header. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + """ salt1_match = self.header.salt1 == self.wal.header.salt1 salt2_match = self.header.salt2 == self.wal.header.salt2 return salt1_match and salt2_match + def validate_checksum(self) -> bool: + """Check if the frame's checksum matches the calculated checksum. + + The checksum values in the final 8 bytes of the frame-header (checksum-1 and checksum-2) + exactly match the computed checksum over: + + 1. the first 24 bytes of the WAL header + 2. the first 8 bytes of each frame header (up to and including this frame) + 3. the page data of each frame (up to and including this frame) + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 + """ + checksum_match = False + base_position = self.fh.tell() + try: + # Read the WAL header bytes from the beginning of the file + wal_hdr_size = len(c_sqlite3.wal_header) + wal_hdr_bytes = self.wal.header.dumps() + if len(wal_hdr_bytes) < wal_hdr_size: + raise EOFError("WAL header too small for checksum calculation") + + # Start seed with checksum over first 24 bytes of WAL header + seed = calculate_checksum(wal_hdr_bytes[:24], endian=self.wal.checksum_endian) + + # Iterate frames from the first frame up to and including this frame + frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size + first_frame_offset = len(c_sqlite3.wal_header) + offset = first_frame_offset + + while offset <= self.offset: + # Read frame header + self.fh.seek(offset) + frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame)) + if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame): + raise EOFError("Incomplete frame header while calculating checksum") + + # Checksum first 16 bytes of frame header + seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian) + + # Read and checksum page data + page_offset = offset + len(c_sqlite3.wal_frame) + self.fh.seek(page_offset) + page_data = self.fh.read(self.wal.header.page_size) + if len(page_data) < self.wal.header.page_size: + raise EOFError("Incomplete page data while calculating checksum") + seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian) + + offset += frame_size + + # Compare calculated checksum to stored checksum in this frame header + checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2) + + finally: + # restore file position + try: + self.fh.seek(base_position) + except Exception: + pass + + return checksum_match + @property def data(self) -> bytes: self.fh.seek(self.offset + len(c_sqlite3.wal_frame)) @@ -187,8 +266,14 @@ class Commit(_FrameCollection): """ -def checksum(buf: bytes, endian: str = ">") -> tuple[int, int]: - s0 = s1 = 0 +def calculate_checksum(buf: bytes, seed: tuple[int, int] = (0, 0), endian: str = ">") -> tuple[int, int]: + """Calculate the checksum of a WAL header or frame. + + References: + - https://sqlite.org/fileformat2.html#checksum_algorithm + """ + + s0, s1 = seed num_ints = len(buf) // 4 arr = struct.unpack(f"{endian}{num_ints}I", buf) diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py index 6d477fe..57ca74c 100644 --- a/tests/sqlite3/test_wal.py +++ b/tests/sqlite3/test_wal.py @@ -18,7 +18,7 @@ ("wal_as_path"), [pytest.param(True, id="wal_as_path"), pytest.param(False, id="wal_as_fh")], ) -def test_sqlite_wal(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None: +def test_sqlite_wal_checkpoint(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None: db = sqlite3.SQLite3( sqlite_db if db_as_path else sqlite_db.open("rb"), sqlite_wal if wal_as_path else sqlite_wal.open("rb"), @@ -47,6 +47,40 @@ def test_sqlite_wal(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_ db.close() +@pytest.mark.parametrize( + ("db_as_path"), + [pytest.param(True, id="db_as_path"), pytest.param(False, id="db_as_fh")], +) +@pytest.mark.parametrize( + ("wal_as_path"), + [pytest.param(True, id="wal_as_path"), pytest.param(False, id="wal_as_fh")], +) +def test_sqlite_wal_checksum_validation(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None: + # Test that the WAL checksum validation works as expected + # When validate_checksum=True, only entries before the last checkpoint are visible + db = sqlite3.SQLite3( + sqlite_db if db_as_path else sqlite_db.open("rb"), + sqlite_wal if wal_as_path else sqlite_wal.open("rb"), + validate_checksum=True, + ) + + _assert_valid_checksum(db) + + db.close() + + # When validate_checksum=False, entries after the last checkpoint are also visible + db = sqlite3.SQLite3( + sqlite_db if db_as_path else sqlite_db.open("rb"), + sqlite_wal if wal_as_path else sqlite_wal.open("rb"), + validate_checksum=False, + ) + + _assert_invalid_checksum(db) + + db.close() + + +# Assertion functions for test_sqlite_wal_checkpoint() def _assert_checkpoint_1(s: sqlite3.SQLite3) -> None: # After the first checkpoint the "after checkpoint" entries are present table = next(iter(s.tables())) @@ -162,3 +196,85 @@ def _assert_checkpoint_3(s: sqlite3.SQLite3) -> None: assert rows[9].id == 11 assert rows[9].name == "second checkpoint" assert rows[9].value == 101 + + +# Assertion functions for test_sqlite_wal_checksum_validation() +def _assert_valid_checksum(s: sqlite3.SQLite3) -> None: + # If the checksum validation is correct, all entries BEFORE the last checkpoint should be present + table = next(iter(s.tables())) + rows = list(table.rows()) + + assert len(rows) == 11 + + assert rows[0].id == 1 + assert rows[0].name == "testing" + assert rows[0].value == 1337 + assert rows[1].id == 2 + assert rows[1].name == "omg" + assert rows[1].value == 7331 + assert rows[2].id == 3 + assert rows[2].name == "A" * 4100 + assert rows[2].value == 4100 + assert rows[3].id == 4 + assert rows[3].name == "B" * 4100 + assert rows[3].value == 4100 + assert rows[4].id == 5 + assert rows[4].name == "negative" + assert rows[4].value == -11644473429 + assert rows[5].id == 6 + assert rows[5].name == "after checkpoint" + assert rows[5].value == 42 + assert rows[6].id == 7 + assert rows[6].name == "after checkpoint" + assert rows[6].value == 43 + assert rows[7].id == 8 + assert rows[7].name == "after checkpoint" + assert rows[7].value == 44 + assert rows[8].id == 9 + assert rows[8].name == "after checkpoint" + assert rows[8].value == 45 + assert rows[9].id == 10 + assert rows[9].name == "second checkpoint" + assert rows[9].value == 100 + assert rows[10].id == 11 + assert rows[10].name == "second checkpoint" + assert rows[10].value == 101 + + +def _assert_invalid_checksum(s: sqlite3.SQLite3) -> None: + # If the checksum validation is incorrect, all entries AFTER the last checkpoint should be present + table = next(iter(s.tables())) + rows = list(table.rows()) + + assert len(rows) == 10 + + assert rows[0].id == 1 + assert rows[0].name == "testing" + assert rows[0].value == 1337 + assert rows[1].id == 2 + assert rows[1].name == "omg" + assert rows[1].value == 7331 + assert rows[2].id == 3 + assert rows[2].name == "A" * 4100 + assert rows[2].value == 4100 + assert rows[3].id == 4 + assert rows[3].name == "B" * 4100 + assert rows[3].value == 4100 + assert rows[4].id == 5 + assert rows[4].name == "negative" + assert rows[4].value == -11644473429 + assert rows[5].id == 6 + assert rows[5].name == "after checkpoint" + assert rows[5].value == 42 + assert rows[6].id == 8 + assert rows[6].name == "after checkpoint" + assert rows[6].value == 44 + assert rows[7].id == 9 + assert rows[7].name == "wow" + assert rows[7].value == 1234 + assert rows[8].id == 10 + assert rows[8].name == "second checkpoint" + assert rows[8].value == 100 + assert rows[9].id == 11 + assert rows[9].name == "second checkpoint" + assert rows[9].value == 101 From e62cf3c20b57a1358437296310e61d47c6b9934f Mon Sep 17 00:00:00 2001 From: Pim <36573021+PimSanders@users.noreply.github.com> Date: Wed, 18 Feb 2026 21:28:33 +0100 Subject: [PATCH 02/19] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/database/sqlite3/wal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 028ddb2..baa5dba 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -137,8 +137,8 @@ def valid(self, validate_checksum: bool = True) -> bool: """ return self.validate_salt() and self.validate_checksum() if validate_checksum else self.validate_salt() - def validate_salt(self) -> bool: - """Check if the frame's salt values match those in the WAL header. + def is_valid_salt(self) -> bool: + """Return whether the frame's salt values match those in the WAL header. References: - https://sqlite.org/fileformat2.html#wal_file_format @@ -148,8 +148,8 @@ def validate_salt(self) -> bool: return salt1_match and salt2_match - def validate_checksum(self) -> bool: - """Check if the frame's checksum matches the calculated checksum. + def is_valid_checksum(self) -> bool: + """Return whether the frame's checksum matches the calculated checksum. The checksum values in the final 8 bytes of the frame-header (checksum-1 and checksum-2) exactly match the computed checksum over: From 3a9927968502be63561d1430233f45e5a7192c5e Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Wed, 18 Feb 2026 21:38:18 +0100 Subject: [PATCH 03/19] Apply suggestions from code review --- dissect/database/sqlite3/sqlite3.py | 3 ++- dissect/database/sqlite3/wal.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py index ca19856..e8c171f 100644 --- a/dissect/database/sqlite3/sqlite3.py +++ b/dissect/database/sqlite3/sqlite3.py @@ -66,6 +66,7 @@ class SQLite3: fh: The path or file-like object to open a SQLite3 database on. wal: The path or file-like object to open a SQLite3 WAL file on. checkpoint: The checkpoint to apply from the WAL file. Can be a :class:`Checkpoint` object or an integer index. + validate_checksum: A boolean that sets whether to validate the checksum of frames when reading. Raises: InvalidDatabase: If the file-like object does not look like a SQLite3 database based on the header magic. @@ -213,7 +214,7 @@ def raw_page(self, num: int) -> bytes: # Check if the latest valid instance of the page is committed (either the frame itself # is the commit frame or it is included in a commit's frames). If so, return that frame's data. for commit in reversed(self.wal.commits): - if (frame := commit.get(num)) and frame.valid(validate_checksum=self.validate_checksum): + if (frame := commit.get(num)) and frame.is_valid(validate_checksum=self.validate_checksum): data = frame.data break diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index baa5dba..0204ebe 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -125,8 +125,8 @@ def __init__(self, wal: WAL, offset: int): def __repr__(self) -> str: return f"" - def valid(self, validate_checksum: bool = True) -> bool: - """Check if the frame is valid by comparing its salt values and optionally verifying the checksum. + def is_valid(self, validate_checksum: bool = True) -> bool: + """Return whether the frame is valid by comparing its salt values and optionally verifying the checksum. A frame is valid if: - Its salt1 and salt2 values match those in the WAL header. @@ -135,7 +135,7 @@ def valid(self, validate_checksum: bool = True) -> bool: References: - https://sqlite.org/fileformat2.html#wal_file_format """ - return self.validate_salt() and self.validate_checksum() if validate_checksum else self.validate_salt() + return self.is_valid_salt() and self.is_valid_checksum() if validate_checksum else self.is_valid_salt() def is_valid_salt(self) -> bool: """Return whether the frame's salt values match those in the WAL header. From 64470ddaceea239740a22b7c44fcc68944ba720a Mon Sep 17 00:00:00 2001 From: Pim <36573021+PimSanders@users.noreply.github.com> Date: Thu, 19 Feb 2026 07:33:49 +0100 Subject: [PATCH 04/19] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/database/sqlite3/sqlite3.py | 3 ++- dissect/database/sqlite3/wal.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py index e8c171f..fa6d0fe 100644 --- a/dissect/database/sqlite3/sqlite3.py +++ b/dissect/database/sqlite3/sqlite3.py @@ -80,7 +80,8 @@ def __init__( fh: Path | BinaryIO, wal: WAL | Path | BinaryIO | None = None, checkpoint: Checkpoint | int | None = None, - validate_checksum: bool = False, + *, + validate_checksums: bool = False, ): if isinstance(fh, Path): path = fh diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 0204ebe..3d9b9f1 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -135,7 +135,7 @@ def is_valid(self, validate_checksum: bool = True) -> bool: References: - https://sqlite.org/fileformat2.html#wal_file_format """ - return self.is_valid_salt() and self.is_valid_checksum() if validate_checksum else self.is_valid_salt() + return (self.is_valid_salt() and self.is_valid_checksum()) if validate_checksum else self.is_valid_salt() def is_valid_salt(self) -> bool: """Return whether the frame's salt values match those in the WAL header. From 4572811991a2d7a835c5ae190b4abc680f40f672 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Thu, 19 Feb 2026 07:38:19 +0100 Subject: [PATCH 05/19] Apply suggestions from code review --- dissect/database/sqlite3/sqlite3.py | 6 +-- dissect/database/sqlite3/wal.py | 78 ++++++++++++----------------- tests/sqlite3/test_wal.py | 8 +-- 3 files changed, 38 insertions(+), 54 deletions(-) diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py index fa6d0fe..1528ba7 100644 --- a/dissect/database/sqlite3/sqlite3.py +++ b/dissect/database/sqlite3/sqlite3.py @@ -66,7 +66,7 @@ class SQLite3: fh: The path or file-like object to open a SQLite3 database on. wal: The path or file-like object to open a SQLite3 WAL file on. checkpoint: The checkpoint to apply from the WAL file. Can be a :class:`Checkpoint` object or an integer index. - validate_checksum: A boolean that sets whether to validate the checksum of frames when reading. + validate_checksums: A boolean that sets whether to validate the checksum of frames when reading. Raises: InvalidDatabase: If the file-like object does not look like a SQLite3 database based on the header magic. @@ -93,7 +93,7 @@ def __init__( self.path = path self.wal = None self.checkpoint = None - self.validate_checksum = validate_checksum + self.validate_checksums = validate_checksums self.header = c_sqlite3.header(self.fh) if self.header.magic != SQLITE3_HEADER_MAGIC: @@ -215,7 +215,7 @@ def raw_page(self, num: int) -> bytes: # Check if the latest valid instance of the page is committed (either the frame itself # is the commit frame or it is included in a commit's frames). If so, return that frame's data. for commit in reversed(self.wal.commits): - if (frame := commit.get(num)) and frame.is_valid(validate_checksum=self.validate_checksum): + if (frame := commit.get(num)) and frame.is_valid(validate_checksums=self.validate_checksums): data = frame.data break diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 3d9b9f1..87a982a 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -125,7 +125,7 @@ def __init__(self, wal: WAL, offset: int): def __repr__(self) -> str: return f"" - def is_valid(self, validate_checksum: bool = True) -> bool: + def is_valid(self, validate_checksums: bool = True) -> bool: """Return whether the frame is valid by comparing its salt values and optionally verifying the checksum. A frame is valid if: @@ -135,7 +135,7 @@ def is_valid(self, validate_checksum: bool = True) -> bool: References: - https://sqlite.org/fileformat2.html#wal_file_format """ - return (self.is_valid_salt() and self.is_valid_checksum()) if validate_checksum else self.is_valid_salt() + return (self.is_valid_salt() and self.is_valid_checksum()) if validate_checksums else self.is_valid_salt() def is_valid_salt(self) -> bool: """Return whether the frame's salt values match those in the WAL header. @@ -163,51 +163,35 @@ def is_valid_checksum(self) -> bool: - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 """ checksum_match = False - base_position = self.fh.tell() - try: - # Read the WAL header bytes from the beginning of the file - wal_hdr_size = len(c_sqlite3.wal_header) - wal_hdr_bytes = self.wal.header.dumps() - if len(wal_hdr_bytes) < wal_hdr_size: - raise EOFError("WAL header too small for checksum calculation") - - # Start seed with checksum over first 24 bytes of WAL header - seed = calculate_checksum(wal_hdr_bytes[:24], endian=self.wal.checksum_endian) - - # Iterate frames from the first frame up to and including this frame - frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size - first_frame_offset = len(c_sqlite3.wal_header) - offset = first_frame_offset - - while offset <= self.offset: - # Read frame header - self.fh.seek(offset) - frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame)) - if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame): - raise EOFError("Incomplete frame header while calculating checksum") - - # Checksum first 16 bytes of frame header - seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian) - - # Read and checksum page data - page_offset = offset + len(c_sqlite3.wal_frame) - self.fh.seek(page_offset) - page_data = self.fh.read(self.wal.header.page_size) - if len(page_data) < self.wal.header.page_size: - raise EOFError("Incomplete page data while calculating checksum") - seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian) - - offset += frame_size - - # Compare calculated checksum to stored checksum in this frame header - checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2) - - finally: - # restore file position - try: - self.fh.seek(base_position) - except Exception: - pass + + # Start seed with checksum over first 24 bytes of WAL header + seed = calculate_checksum(self.header.dumps()[:24], endian=self.wal.checksum_endian) + + # Iterate frames from the first frame up to and including this frame + frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size + first_frame_offset = len(c_sqlite3.wal_header) + offset = first_frame_offset + + while offset <= self.offset: + # Read frame header + self.fh.seek(offset) + frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame)) + if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame): + raise EOFError("Incomplete frame header while calculating checksum") + + # Checksum first 16 bytes of frame header + seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian) + + # Read and checksum page data + page_data = self.fh.read(self.wal.header.page_size) + if len(page_data) < self.wal.header.page_size: + raise EOFError("Incomplete page data while calculating checksum") + seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian) + + offset += frame_size + + # Compare calculated checksum to stored checksum in this frame header + checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2) return checksum_match diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py index 57ca74c..46fe0b3 100644 --- a/tests/sqlite3/test_wal.py +++ b/tests/sqlite3/test_wal.py @@ -57,22 +57,22 @@ def test_sqlite_wal_checkpoint(sqlite_db: Path, sqlite_wal: Path, db_as_path: bo ) def test_sqlite_wal_checksum_validation(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None: # Test that the WAL checksum validation works as expected - # When validate_checksum=True, only entries before the last checkpoint are visible + # When validate_checksums=True, only entries before the last checkpoint are visible db = sqlite3.SQLite3( sqlite_db if db_as_path else sqlite_db.open("rb"), sqlite_wal if wal_as_path else sqlite_wal.open("rb"), - validate_checksum=True, + validate_checksums=True, ) _assert_valid_checksum(db) db.close() - # When validate_checksum=False, entries after the last checkpoint are also visible + # When validate_checksums=False, entries after the last checkpoint are also visible db = sqlite3.SQLite3( sqlite_db if db_as_path else sqlite_db.open("rb"), sqlite_wal if wal_as_path else sqlite_wal.open("rb"), - validate_checksum=False, + validate_checksums=False, ) _assert_invalid_checksum(db) From f4b6ffb07c113f876b1e91c84d8982545e9cae66 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Thu, 19 Feb 2026 07:42:53 +0100 Subject: [PATCH 06/19] Fix linter --- dissect/database/sqlite3/wal.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 87a982a..b40799f 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -162,8 +162,6 @@ def is_valid_checksum(self) -> bool: - https://sqlite.org/fileformat2.html#wal_file_format - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 """ - checksum_match = False - # Start seed with checksum over first 24 bytes of WAL header seed = calculate_checksum(self.header.dumps()[:24], endian=self.wal.checksum_endian) @@ -191,9 +189,7 @@ def is_valid_checksum(self) -> bool: offset += frame_size # Compare calculated checksum to stored checksum in this frame header - checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2) - - return checksum_match + return (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2) @property def data(self) -> bytes: From 20ffbfb5a67b93709641bc9cd334719f091b743e Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Wed, 27 May 2026 21:03:57 +0200 Subject: [PATCH 07/19] Add benchmark --- dissect/database/sqlite3/wal.py | 5 +++-- tests/_data/sqlite3/big.sqlite | 3 +++ tests/_data/sqlite3/big.sqlite-wal | 3 +++ tests/sqlite3/conftest.py | 10 ++++++++++ tests/sqlite3/test_wal.py | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 tests/_data/sqlite3/big.sqlite create mode 100644 tests/_data/sqlite3/big.sqlite-wal diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index c37dbc3..2261fe8 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -39,7 +39,9 @@ def __init__(self, fh: Path | BinaryIO): raise InvalidDatabase("Invalid WAL header magic") self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">" - self.highest_page_num = max(fr.page_number for commit in self.commits for fr in commit.frames if fr.valid) + self.highest_page_num = max( + fr.page_number for commit in self.commits for fr in commit.frames if fr.is_valid_salt() + ) self.frame = lru_cache(1024)(self.frame) @@ -253,7 +255,6 @@ def calculate_checksum(buf: bytes, seed: tuple[int, int] = (0, 0), endian: str = References: - https://sqlite.org/fileformat2.html#checksum_algorithm """ - s0, s1 = seed num_ints = len(buf) // 4 arr = struct.unpack(f"{endian}{num_ints}I", buf) diff --git a/tests/_data/sqlite3/big.sqlite b/tests/_data/sqlite3/big.sqlite new file mode 100644 index 0000000..bb2cb0f --- /dev/null +++ b/tests/_data/sqlite3/big.sqlite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49abb18da561d667cd02cd3f4aa5ad6f80740f29f9c2a24b6e7286cbe259ffe2 +size 69632 diff --git a/tests/_data/sqlite3/big.sqlite-wal b/tests/_data/sqlite3/big.sqlite-wal new file mode 100644 index 0000000..2df6fe5 --- /dev/null +++ b/tests/_data/sqlite3/big.sqlite-wal @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ada96856785eed8f8aca40d43f788cdb515eed6fcd40b77004134471c9160fd8 +size 8305952 diff --git a/tests/sqlite3/conftest.py b/tests/sqlite3/conftest.py index 9f86947..3579c0f 100644 --- a/tests/sqlite3/conftest.py +++ b/tests/sqlite3/conftest.py @@ -23,3 +23,13 @@ def sqlite_wal() -> Path: @pytest.fixture def empty_db() -> Path: return absolute_path("_data/sqlite3/empty.sqlite") + + +@pytest.fixture +def big_sqlite_db() -> Path: + return absolute_path("_data/sqlite3/big.sqlite") + + +@pytest.fixture +def big_sqlite_wal() -> Path: + return absolute_path("_data/sqlite3/big.sqlite-wal") diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py index 5e3b281..d7f91b9 100644 --- a/tests/sqlite3/test_wal.py +++ b/tests/sqlite3/test_wal.py @@ -10,6 +10,8 @@ if TYPE_CHECKING: from pathlib import Path + from pytest_benchmark.fixture import BenchmarkFixture + @pytest.mark.parametrize( ("db_as_path"), @@ -314,3 +316,19 @@ def test_wal_page_count() -> None: assert db.wal.highest_page_num == 4 assert db.header.page_count == 2 assert db.page_count == 4 + + +@pytest.mark.parametrize( + ("validate"), + [pytest.param(True, id="True"), pytest.param(False, id="False")], +) +@pytest.mark.benchmark +def test_benchmark_wal_checksum_validation( + big_sqlite_db: Path, big_sqlite_wal: Path, validate: bool, benchmark: BenchmarkFixture +) -> None: + def benchy() -> None: + # list(db.tables()[0].rows()) + db = sqlite3.SQLite3(big_sqlite_db, big_sqlite_wal, validate_checksums=validate) + list(list(db.tables())[0].rows()) + + benchmark(benchy) From 5d0cbc732e8ba9f306167c91ab8ccd4ffdbefe43 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Wed, 27 May 2026 21:07:00 +0200 Subject: [PATCH 08/19] Fix linter again, as usual --- tests/sqlite3/test_wal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py index d7f91b9..5db5fac 100644 --- a/tests/sqlite3/test_wal.py +++ b/tests/sqlite3/test_wal.py @@ -329,6 +329,6 @@ def test_benchmark_wal_checksum_validation( def benchy() -> None: # list(db.tables()[0].rows()) db = sqlite3.SQLite3(big_sqlite_db, big_sqlite_wal, validate_checksums=validate) - list(list(db.tables())[0].rows()) + list(next(iter(db.tables()))) benchmark(benchy) From c70d440543adb9904787376d10b88e90072996ab Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Fri, 29 May 2026 21:56:48 +0200 Subject: [PATCH 09/19] Remove commented line --- tests/sqlite3/test_wal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py index 5db5fac..d3ffdd5 100644 --- a/tests/sqlite3/test_wal.py +++ b/tests/sqlite3/test_wal.py @@ -327,7 +327,6 @@ def test_benchmark_wal_checksum_validation( big_sqlite_db: Path, big_sqlite_wal: Path, validate: bool, benchmark: BenchmarkFixture ) -> None: def benchy() -> None: - # list(db.tables()[0].rows()) db = sqlite3.SQLite3(big_sqlite_db, big_sqlite_wal, validate_checksums=validate) list(next(iter(db.tables()))) From ba48643a61be84a1c593dd6e0ac40953b5df649b Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Fri, 29 May 2026 22:16:09 +0200 Subject: [PATCH 10/19] Cache initial checksum from WAL header --- dissect/database/sqlite3/wal.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 2261fe8..6f4f276 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -114,6 +114,11 @@ def checkpoints(self) -> list[Checkpoint]: return [checkpoints_map[salt] for salt in sorted(checkpoints_map.keys())] + @cached_property + def header_checksum_seed(self) -> tuple[int, int]: + """Cached initial checksum seed calculated from the WAL header first 24 bytes.""" + return calculate_checksum(self.header.dumps()[:24], endian=self.checksum_endian) + class Frame: def __init__(self, wal: WAL, offset: int): @@ -165,8 +170,8 @@ def is_valid_checksum(self) -> bool: - https://sqlite.org/fileformat2.html#wal_file_format - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 """ - # Start seed with checksum over first 24 bytes of WAL header - seed = calculate_checksum(self.header.dumps()[:24], endian=self.wal.checksum_endian) + # Start seed with checksum over first 24 bytes of WAL header (cached on WAL) + seed = self.wal.header_checksum_seed # Iterate frames from the first frame up to and including this frame frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size From a335ccc9a6dc124ceee46f3698fd8b931d1af711 Mon Sep 17 00:00:00 2001 From: Pim <36573021+PimSanders@users.noreply.github.com> Date: Fri, 29 May 2026 22:18:12 +0200 Subject: [PATCH 11/19] Update dissect/database/sqlite3/wal.py Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/database/sqlite3/wal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 6f4f276..c289cc8 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -163,7 +163,7 @@ def is_valid_checksum(self) -> bool: exactly match the computed checksum over: 1. the first 24 bytes of the WAL header - 2. the first 8 bytes of each frame header (up to and including this frame) + 2. the first 16 bytes of each frame header (up to and including this frame) 3. the page data of each frame (up to and including this frame) References: From 7400171d6f73ae11b468d946fb94c7d0cb5783fb Mon Sep 17 00:00:00 2001 From: Pim <36573021+PimSanders@users.noreply.github.com> Date: Sun, 21 Jun 2026 15:28:28 +0200 Subject: [PATCH 12/19] Use last seed method --- dissect/database/sqlite3/wal.py | 119 +++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index c289cc8..2ce3a65 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -39,12 +39,24 @@ def __init__(self, fh: Path | BinaryIO): raise InvalidDatabase("Invalid WAL header magic") self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">" + + self.frame = lru_cache(1024)(self.frame) + self.frame_size = len(c_sqlite3.wal_frame) + self.header.page_size + self.first_frame_offset = len(c_sqlite3.wal_header) + + # Only track the highest valid offset and its seed. + # Meaning: all frames with offset < _highest_valid_next_offset are considered valid. + # _highest_valid_next_offset initially points at the first frame; seed is checksum over header. + self._highest_valid_next_offset: int = self.first_frame_offset + self._highest_valid_seed: tuple[int, int] = self.header_checksum_seed + + # First offset that is known to fail checksum validation, or None. + self._checksum_failed_offset: int | None = None + self.highest_page_num = max( fr.page_number for commit in self.commits for fr in commit.frames if fr.is_valid_salt() ) - self.frame = lru_cache(1024)(self.frame) - def close(self) -> None: """Close the WAL.""" # Only close WAL handle if we opened it using a path @@ -52,8 +64,7 @@ def close(self) -> None: self.fh.close() def frame(self, frame_idx: int) -> Frame: - frame_size = len(c_sqlite3.wal_frame) + self.header.page_size - offset = len(c_sqlite3.wal_header) + frame_idx * frame_size + offset = self.first_frame_offset + frame_idx * self.frame_size return Frame(self, offset) def frames(self) -> Iterator[Frame]: @@ -65,6 +76,62 @@ def frames(self) -> Iterator[Frame]: except EOFError: # noqa: PERF203 break + def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None: + """Return checksum seed after processing frames up to and including the frame at target_offset. + + If validate=True, verify stored checksums for each frame as we walk. If a mismatch is found, update + the WAL's highest-known-valid-next-offset and return None. On success (no mismatches) update + the highest-known-valid-next-offset and seed and return the computed seed. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 + """ + # If the target offset is before the first frame, return the initial seed calculated from the WAL header. + if target_offset < self.first_frame_offset: + return self.header_checksum_seed + + # Start from the highest verified offset we know (saves re-checking earlier frames). + base_offset = ( + self._highest_valid_next_offset + if self._highest_valid_next_offset <= target_offset + else self.first_frame_offset + ) + seed = self._highest_valid_seed if base_offset == self._highest_valid_next_offset else self.header_checksum_seed + offset = base_offset + + while offset <= target_offset: + # Read frame header + self.fh.seek(offset) + frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame)) + if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame): + raise EOFError("Incomplete frame header while calculating checksum") + + # Checksum first 16 bytes of frame header + seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.checksum_endian) + + # Read and checksum page data + page_data = self.fh.read(self.header.page_size) + if len(page_data) < self.header.page_size: + raise EOFError("Incomplete page data while calculating checksum") + seed = calculate_checksum(page_data, seed=seed, endian=self.checksum_endian) + + # Compare computed seed to stored checksums in this frame header. + checksum1, checksum2 = struct.unpack(f"{self.checksum_endian}2I", frame_hdr_bytes[-8:]) + if (seed[0], seed[1]) != (checksum1, checksum2): + self._highest_valid_next_offset = min(self._highest_valid_next_offset, offset) + self._checksum_failed_offset = offset + + return None + + offset += self.frame_size + + # Update highest-known-valid-next-offset and seed to the next offset after target. + self._highest_valid_next_offset = offset + self._highest_valid_seed = seed + + return seed + @cached_property def commits(self) -> list[Commit]: """Return all commits in the WAL file. @@ -159,45 +226,15 @@ def is_valid_salt(self) -> bool: def is_valid_checksum(self) -> bool: """Return whether the frame's checksum matches the calculated checksum. - The checksum values in the final 8 bytes of the frame-header (checksum-1 and checksum-2) - exactly match the computed checksum over: - - 1. the first 24 bytes of the WAL header - 2. the first 16 bytes of each frame header (up to and including this frame) - 3. the page data of each frame (up to and including this frame) - - References: - - https://sqlite.org/fileformat2.html#wal_file_format - - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 + Use WAL's highest valid offset to skip checks for already-verified frames. """ - # Start seed with checksum over first 24 bytes of WAL header (cached on WAL) - seed = self.wal.header_checksum_seed - - # Iterate frames from the first frame up to and including this frame - frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size - first_frame_offset = len(c_sqlite3.wal_header) - offset = first_frame_offset - - while offset <= self.offset: - # Read frame header - self.fh.seek(offset) - frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame)) - if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame): - raise EOFError("Incomplete frame header while calculating checksum") - - # Checksum first 16 bytes of frame header - seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian) - - # Read and checksum page data - page_data = self.fh.read(self.wal.header.page_size) - if len(page_data) < self.wal.header.page_size: - raise EOFError("Incomplete page data while calculating checksum") - seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian) - - offset += frame_size + if self.offset < self.wal._highest_valid_next_offset: + return True + if self.wal._checksum_failed_offset is not None and self.offset >= self.wal._checksum_failed_offset: + return False - # Compare calculated checksum to stored checksum in this frame header - return (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2) + seed = self.wal.seed_for_offset(self.offset) + return seed is not None @property def data(self) -> bytes: From 92a6b9cd4ef01193ce77fb63c71a4e41421375a6 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:40:29 +0200 Subject: [PATCH 13/19] Rename vars --- dissect/database/sqlite3/wal.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 2ce3a65..aa92f17 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -76,7 +76,7 @@ def frames(self) -> Iterator[Frame]: except EOFError: # noqa: PERF203 break - def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None: + def seed_for_offset(self, offset: int) -> tuple[int, int] | None: """Return checksum seed after processing frames up to and including the frame at target_offset. If validate=True, verify stored checksums for each frame as we walk. If a mismatch is found, update @@ -88,21 +88,21 @@ def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None: - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047 """ # If the target offset is before the first frame, return the initial seed calculated from the WAL header. - if target_offset < self.first_frame_offset: + if offset < self.first_frame_offset: return self.header_checksum_seed # Start from the highest verified offset we know (saves re-checking earlier frames). base_offset = ( self._highest_valid_next_offset - if self._highest_valid_next_offset <= target_offset + if self._highest_valid_next_offset <= offset else self.first_frame_offset ) seed = self._highest_valid_seed if base_offset == self._highest_valid_next_offset else self.header_checksum_seed - offset = base_offset + current_offset = base_offset - while offset <= target_offset: + while current_offset <= offset: # Read frame header - self.fh.seek(offset) + self.fh.seek(current_offset) frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame)) if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame): raise EOFError("Incomplete frame header while calculating checksum") @@ -119,15 +119,15 @@ def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None: # Compare computed seed to stored checksums in this frame header. checksum1, checksum2 = struct.unpack(f"{self.checksum_endian}2I", frame_hdr_bytes[-8:]) if (seed[0], seed[1]) != (checksum1, checksum2): - self._highest_valid_next_offset = min(self._highest_valid_next_offset, offset) - self._checksum_failed_offset = offset + self._highest_valid_next_offset = min(self._highest_valid_next_offset, current_offset) + self._checksum_failed_offset = current_offset return None - offset += self.frame_size + current_offset += self.frame_size # Update highest-known-valid-next-offset and seed to the next offset after target. - self._highest_valid_next_offset = offset + self._highest_valid_next_offset = current_offset self._highest_valid_seed = seed return seed From 0b72b703941e833318381719111a33b56b0c262b Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:43:28 +0200 Subject: [PATCH 14/19] Update comment --- dissect/database/sqlite3/wal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index aa92f17..2a26fa0 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -79,9 +79,9 @@ def frames(self) -> Iterator[Frame]: def seed_for_offset(self, offset: int) -> tuple[int, int] | None: """Return checksum seed after processing frames up to and including the frame at target_offset. - If validate=True, verify stored checksums for each frame as we walk. If a mismatch is found, update - the WAL's highest-known-valid-next-offset and return None. On success (no mismatches) update - the highest-known-valid-next-offset and seed and return the computed seed. + Verify stored checksums for each frame as we walk. If a mismatch is found, update the WAL's + highest-known-valid-next-offset and return None. On success (no mismatches) update the + highest-known-valid-next-offset and seed and return the computed seed. References: - https://sqlite.org/fileformat2.html#wal_file_format From d0f5d4fb1dce1b37d56955b8928650ce2e4b8a6f Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:51:07 +0200 Subject: [PATCH 15/19] Remove redundant if statemnets --- dissect/database/sqlite3/wal.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 2a26fa0..5ea87a5 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -92,13 +92,8 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None: return self.header_checksum_seed # Start from the highest verified offset we know (saves re-checking earlier frames). - base_offset = ( - self._highest_valid_next_offset - if self._highest_valid_next_offset <= offset - else self.first_frame_offset - ) - seed = self._highest_valid_seed if base_offset == self._highest_valid_next_offset else self.header_checksum_seed - current_offset = base_offset + current_offset = self._highest_valid_next_offset + seed = self._highest_valid_seed while current_offset <= offset: # Read frame header From 61758d90da1b1fa1a306c8f2c59910bd6e10f1ea Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:56:25 +0200 Subject: [PATCH 16/19] Use precompiled struct --- dissect/database/sqlite3/wal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 5ea87a5..bc59520 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -39,6 +39,7 @@ def __init__(self, fh: Path | BinaryIO): raise InvalidDatabase("Invalid WAL header magic") self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">" + self._checksum_struct = struct.Struct(f"{self.checksum_endian}2I") self.frame = lru_cache(1024)(self.frame) self.frame_size = len(c_sqlite3.wal_frame) + self.header.page_size @@ -112,7 +113,7 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None: seed = calculate_checksum(page_data, seed=seed, endian=self.checksum_endian) # Compare computed seed to stored checksums in this frame header. - checksum1, checksum2 = struct.unpack(f"{self.checksum_endian}2I", frame_hdr_bytes[-8:]) + checksum1, checksum2 = self._checksum_struct.unpack(frame_hdr_bytes[-8:]) if (seed[0], seed[1]) != (checksum1, checksum2): self._highest_valid_next_offset = min(self._highest_valid_next_offset, current_offset) self._checksum_failed_offset = current_offset From 9bd3e762670ab803aa489934f35caf6a5d6960e1 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:59:39 +0200 Subject: [PATCH 17/19] Do not update self._highest_valid_next_offset when checksum fails --- dissect/database/sqlite3/wal.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index bc59520..9a1650b 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -115,9 +115,7 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None: # Compare computed seed to stored checksums in this frame header. checksum1, checksum2 = self._checksum_struct.unpack(frame_hdr_bytes[-8:]) if (seed[0], seed[1]) != (checksum1, checksum2): - self._highest_valid_next_offset = min(self._highest_valid_next_offset, current_offset) self._checksum_failed_offset = current_offset - return None current_offset += self.frame_size From 114bdc20355b785be2959043ca29a5c6f78e5c45 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:59:51 +0200 Subject: [PATCH 18/19] Move failed offset check to seed_for_offset --- dissect/database/sqlite3/wal.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py index 9a1650b..04a538a 100644 --- a/dissect/database/sqlite3/wal.py +++ b/dissect/database/sqlite3/wal.py @@ -92,6 +92,10 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None: if offset < self.first_frame_offset: return self.header_checksum_seed + # If the target offset is at or beyond the first known checksum failure, return None. + if self._checksum_failed_offset is not None and offset >= self._checksum_failed_offset: + return None + # Start from the highest verified offset we know (saves re-checking earlier frames). current_offset = self._highest_valid_next_offset seed = self._highest_valid_seed @@ -224,8 +228,6 @@ def is_valid_checksum(self) -> bool: """ if self.offset < self.wal._highest_valid_next_offset: return True - if self.wal._checksum_failed_offset is not None and self.offset >= self.wal._checksum_failed_offset: - return False seed = self.wal.seed_for_offset(self.offset) return seed is not None From ec7b77e169117364b92ef64068c0dcacb39d41f8 Mon Sep 17 00:00:00 2001 From: Pim Sanders <36573021+PimSanders@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:30:42 +0200 Subject: [PATCH 19/19] Enable checksum validation by default --- dissect/database/sqlite3/sqlite3.py | 2 +- tests/sqlite3/test_sqlite3.py | 4 ++-- tests/sqlite3/test_wal.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py index 2613ce3..7fdabcd 100644 --- a/dissect/database/sqlite3/sqlite3.py +++ b/dissect/database/sqlite3/sqlite3.py @@ -84,7 +84,7 @@ def __init__( wal: WAL | Path | BinaryIO | None = None, checkpoint: Checkpoint | int | None = None, *, - validate_checksums: bool = False, + validate_checksums: bool = True, ): if isinstance(fh, Path): path = fh diff --git a/tests/sqlite3/test_sqlite3.py b/tests/sqlite3/test_sqlite3.py index f5caaf9..e56a26b 100644 --- a/tests/sqlite3/test_sqlite3.py +++ b/tests/sqlite3/test_sqlite3.py @@ -17,11 +17,11 @@ [pytest.param(True, id="as_path"), pytest.param(False, id="as_fh")], ) def test_sqlite(sqlite_db: Path, open_as_path: bool) -> None: - db = sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb")) + db = sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb"), validate_checksums=False) _assert_sqlite_db(db) db.close() - with sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb")) as db: + with sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb"), validate_checksums=False) as db: _assert_sqlite_db(db) diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py index d3ffdd5..8c71f71 100644 --- a/tests/sqlite3/test_wal.py +++ b/tests/sqlite3/test_wal.py @@ -304,7 +304,7 @@ def test_wal_page_count() -> None: >>> con.commit() # Copy page_count.db* files before closing """ - db = sqlite3.SQLite3(absolute_path("_data/sqlite3/page_count.db")) + db = sqlite3.SQLite3(absolute_path("_data/sqlite3/page_count.db"), validate_checksums=False) table = db.table("t1") assert table.sql == "CREATE TABLE t1 (a, b)"