From 0e3eeb6eb417f4307276d327a5e3b59be32e404b Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Mon, 2 Feb 2026 16:05:56 +0100
Subject: [PATCH 01/19] Expand WAL validation
---
dissect/database/sqlite3/sqlite3.py | 4 +-
dissect/database/sqlite3/wal.py | 93 +++++++++++++++++++++-
tests/sqlite3/test_wal.py | 118 +++++++++++++++++++++++++++-
3 files changed, 209 insertions(+), 6 deletions(-)
diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py
index e009053..ca19856 100644
--- a/dissect/database/sqlite3/sqlite3.py
+++ b/dissect/database/sqlite3/sqlite3.py
@@ -79,6 +79,7 @@ def __init__(
fh: Path | BinaryIO,
wal: WAL | Path | BinaryIO | None = None,
checkpoint: Checkpoint | int | None = None,
+ validate_checksum: bool = False,
):
if isinstance(fh, Path):
path = fh
@@ -90,6 +91,7 @@ def __init__(
self.path = path
self.wal = None
self.checkpoint = None
+ self.validate_checksum = validate_checksum
self.header = c_sqlite3.header(self.fh)
if self.header.magic != SQLITE3_HEADER_MAGIC:
@@ -211,7 +213,7 @@ def raw_page(self, num: int) -> bytes:
# Check if the latest valid instance of the page is committed (either the frame itself
# is the commit frame or it is included in a commit's frames). If so, return that frame's data.
for commit in reversed(self.wal.commits):
- if (frame := commit.get(num)) and frame.valid:
+ if (frame := commit.get(num)) and frame.valid(validate_checksum=self.validate_checksum):
data = frame.data
break
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 7d4ec76..028ddb2 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -125,13 +125,92 @@ def __init__(self, wal: WAL, offset: int):
def __repr__(self) -> str:
return f""
- @property
- def valid(self) -> bool:
+ def valid(self, validate_checksum: bool = True) -> bool:
+ """Check if the frame is valid by comparing its salt values and optionally verifying the checksum.
+
+ A frame is valid if:
+ - Its salt1 and salt2 values match those in the WAL header.
+ - Its checksum matches the calculated checksum.
+
+ References:
+ - https://sqlite.org/fileformat2.html#wal_file_format
+ """
+ return self.validate_salt() and self.validate_checksum() if validate_checksum else self.validate_salt()
+
+ def validate_salt(self) -> bool:
+ """Check if the frame's salt values match those in the WAL header.
+
+ References:
+ - https://sqlite.org/fileformat2.html#wal_file_format
+ """
salt1_match = self.header.salt1 == self.wal.header.salt1
salt2_match = self.header.salt2 == self.wal.header.salt2
return salt1_match and salt2_match
+ def validate_checksum(self) -> bool:
+ """Check if the frame's checksum matches the calculated checksum.
+
+ The checksum values in the final 8 bytes of the frame-header (checksum-1 and checksum-2)
+ exactly match the computed checksum over:
+
+ 1. the first 24 bytes of the WAL header
+ 2. the first 8 bytes of each frame header (up to and including this frame)
+ 3. the page data of each frame (up to and including this frame)
+
+ References:
+ - https://sqlite.org/fileformat2.html#wal_file_format
+ - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
+ """
+ checksum_match = False
+ base_position = self.fh.tell()
+ try:
+ # Read the WAL header bytes from the beginning of the file
+ wal_hdr_size = len(c_sqlite3.wal_header)
+ wal_hdr_bytes = self.wal.header.dumps()
+ if len(wal_hdr_bytes) < wal_hdr_size:
+ raise EOFError("WAL header too small for checksum calculation")
+
+ # Start seed with checksum over first 24 bytes of WAL header
+ seed = calculate_checksum(wal_hdr_bytes[:24], endian=self.wal.checksum_endian)
+
+ # Iterate frames from the first frame up to and including this frame
+ frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size
+ first_frame_offset = len(c_sqlite3.wal_header)
+ offset = first_frame_offset
+
+ while offset <= self.offset:
+ # Read frame header
+ self.fh.seek(offset)
+ frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame))
+ if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame):
+ raise EOFError("Incomplete frame header while calculating checksum")
+
+ # Checksum first 16 bytes of frame header
+ seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian)
+
+ # Read and checksum page data
+ page_offset = offset + len(c_sqlite3.wal_frame)
+ self.fh.seek(page_offset)
+ page_data = self.fh.read(self.wal.header.page_size)
+ if len(page_data) < self.wal.header.page_size:
+ raise EOFError("Incomplete page data while calculating checksum")
+ seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian)
+
+ offset += frame_size
+
+ # Compare calculated checksum to stored checksum in this frame header
+ checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2)
+
+ finally:
+ # restore file position
+ try:
+ self.fh.seek(base_position)
+ except Exception:
+ pass
+
+ return checksum_match
+
@property
def data(self) -> bytes:
self.fh.seek(self.offset + len(c_sqlite3.wal_frame))
@@ -187,8 +266,14 @@ class Commit(_FrameCollection):
"""
-def checksum(buf: bytes, endian: str = ">") -> tuple[int, int]:
- s0 = s1 = 0
+def calculate_checksum(buf: bytes, seed: tuple[int, int] = (0, 0), endian: str = ">") -> tuple[int, int]:
+ """Calculate the checksum of a WAL header or frame.
+
+ References:
+ - https://sqlite.org/fileformat2.html#checksum_algorithm
+ """
+
+ s0, s1 = seed
num_ints = len(buf) // 4
arr = struct.unpack(f"{endian}{num_ints}I", buf)
diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py
index 6d477fe..57ca74c 100644
--- a/tests/sqlite3/test_wal.py
+++ b/tests/sqlite3/test_wal.py
@@ -18,7 +18,7 @@
("wal_as_path"),
[pytest.param(True, id="wal_as_path"), pytest.param(False, id="wal_as_fh")],
)
-def test_sqlite_wal(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None:
+def test_sqlite_wal_checkpoint(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None:
db = sqlite3.SQLite3(
sqlite_db if db_as_path else sqlite_db.open("rb"),
sqlite_wal if wal_as_path else sqlite_wal.open("rb"),
@@ -47,6 +47,40 @@ def test_sqlite_wal(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_
db.close()
+@pytest.mark.parametrize(
+ ("db_as_path"),
+ [pytest.param(True, id="db_as_path"), pytest.param(False, id="db_as_fh")],
+)
+@pytest.mark.parametrize(
+ ("wal_as_path"),
+ [pytest.param(True, id="wal_as_path"), pytest.param(False, id="wal_as_fh")],
+)
+def test_sqlite_wal_checksum_validation(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None:
+ # Test that the WAL checksum validation works as expected
+ # When validate_checksum=True, only entries before the last checkpoint are visible
+ db = sqlite3.SQLite3(
+ sqlite_db if db_as_path else sqlite_db.open("rb"),
+ sqlite_wal if wal_as_path else sqlite_wal.open("rb"),
+ validate_checksum=True,
+ )
+
+ _assert_valid_checksum(db)
+
+ db.close()
+
+ # When validate_checksum=False, entries after the last checkpoint are also visible
+ db = sqlite3.SQLite3(
+ sqlite_db if db_as_path else sqlite_db.open("rb"),
+ sqlite_wal if wal_as_path else sqlite_wal.open("rb"),
+ validate_checksum=False,
+ )
+
+ _assert_invalid_checksum(db)
+
+ db.close()
+
+
+# Assertion functions for test_sqlite_wal_checkpoint()
def _assert_checkpoint_1(s: sqlite3.SQLite3) -> None:
# After the first checkpoint the "after checkpoint" entries are present
table = next(iter(s.tables()))
@@ -162,3 +196,85 @@ def _assert_checkpoint_3(s: sqlite3.SQLite3) -> None:
assert rows[9].id == 11
assert rows[9].name == "second checkpoint"
assert rows[9].value == 101
+
+
+# Assertion functions for test_sqlite_wal_checksum_validation()
+def _assert_valid_checksum(s: sqlite3.SQLite3) -> None:
+ # If the checksum validation is correct, all entries BEFORE the last checkpoint should be present
+ table = next(iter(s.tables()))
+ rows = list(table.rows())
+
+ assert len(rows) == 11
+
+ assert rows[0].id == 1
+ assert rows[0].name == "testing"
+ assert rows[0].value == 1337
+ assert rows[1].id == 2
+ assert rows[1].name == "omg"
+ assert rows[1].value == 7331
+ assert rows[2].id == 3
+ assert rows[2].name == "A" * 4100
+ assert rows[2].value == 4100
+ assert rows[3].id == 4
+ assert rows[3].name == "B" * 4100
+ assert rows[3].value == 4100
+ assert rows[4].id == 5
+ assert rows[4].name == "negative"
+ assert rows[4].value == -11644473429
+ assert rows[5].id == 6
+ assert rows[5].name == "after checkpoint"
+ assert rows[5].value == 42
+ assert rows[6].id == 7
+ assert rows[6].name == "after checkpoint"
+ assert rows[6].value == 43
+ assert rows[7].id == 8
+ assert rows[7].name == "after checkpoint"
+ assert rows[7].value == 44
+ assert rows[8].id == 9
+ assert rows[8].name == "after checkpoint"
+ assert rows[8].value == 45
+ assert rows[9].id == 10
+ assert rows[9].name == "second checkpoint"
+ assert rows[9].value == 100
+ assert rows[10].id == 11
+ assert rows[10].name == "second checkpoint"
+ assert rows[10].value == 101
+
+
+def _assert_invalid_checksum(s: sqlite3.SQLite3) -> None:
+ # If the checksum validation is incorrect, all entries AFTER the last checkpoint should be present
+ table = next(iter(s.tables()))
+ rows = list(table.rows())
+
+ assert len(rows) == 10
+
+ assert rows[0].id == 1
+ assert rows[0].name == "testing"
+ assert rows[0].value == 1337
+ assert rows[1].id == 2
+ assert rows[1].name == "omg"
+ assert rows[1].value == 7331
+ assert rows[2].id == 3
+ assert rows[2].name == "A" * 4100
+ assert rows[2].value == 4100
+ assert rows[3].id == 4
+ assert rows[3].name == "B" * 4100
+ assert rows[3].value == 4100
+ assert rows[4].id == 5
+ assert rows[4].name == "negative"
+ assert rows[4].value == -11644473429
+ assert rows[5].id == 6
+ assert rows[5].name == "after checkpoint"
+ assert rows[5].value == 42
+ assert rows[6].id == 8
+ assert rows[6].name == "after checkpoint"
+ assert rows[6].value == 44
+ assert rows[7].id == 9
+ assert rows[7].name == "wow"
+ assert rows[7].value == 1234
+ assert rows[8].id == 10
+ assert rows[8].name == "second checkpoint"
+ assert rows[8].value == 100
+ assert rows[9].id == 11
+ assert rows[9].name == "second checkpoint"
+ assert rows[9].value == 101
From e62cf3c20b57a1358437296310e61d47c6b9934f Mon Sep 17 00:00:00 2001
From: Pim <36573021+PimSanders@users.noreply.github.com>
Date: Wed, 18 Feb 2026 21:28:33 +0100
Subject: [PATCH 02/19] Apply suggestions from code review
Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com>
---
dissect/database/sqlite3/wal.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 028ddb2..baa5dba 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -137,8 +137,8 @@ def valid(self, validate_checksum: bool = True) -> bool:
"""
return self.validate_salt() and self.validate_checksum() if validate_checksum else self.validate_salt()
- def validate_salt(self) -> bool:
- """Check if the frame's salt values match those in the WAL header.
+ def is_valid_salt(self) -> bool:
+ """Return whether the frame's salt values match those in the WAL header.
References:
- https://sqlite.org/fileformat2.html#wal_file_format
@@ -148,8 +148,8 @@ def validate_salt(self) -> bool:
return salt1_match and salt2_match
- def validate_checksum(self) -> bool:
- """Check if the frame's checksum matches the calculated checksum.
+ def is_valid_checksum(self) -> bool:
+ """Return whether the frame's checksum matches the calculated checksum.
The checksum values in the final 8 bytes of the frame-header (checksum-1 and checksum-2)
exactly match the computed checksum over:
From 3a9927968502be63561d1430233f45e5a7192c5e Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Wed, 18 Feb 2026 21:38:18 +0100
Subject: [PATCH 03/19] Apply suggestions from code review
---
dissect/database/sqlite3/sqlite3.py | 3 ++-
dissect/database/sqlite3/wal.py | 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py
index ca19856..e8c171f 100644
--- a/dissect/database/sqlite3/sqlite3.py
+++ b/dissect/database/sqlite3/sqlite3.py
@@ -66,6 +66,7 @@ class SQLite3:
fh: The path or file-like object to open a SQLite3 database on.
wal: The path or file-like object to open a SQLite3 WAL file on.
checkpoint: The checkpoint to apply from the WAL file. Can be a :class:`Checkpoint` object or an integer index.
+ validate_checksum: A boolean that sets whether to validate the checksum of frames when reading.
Raises:
InvalidDatabase: If the file-like object does not look like a SQLite3 database based on the header magic.
@@ -213,7 +214,7 @@ def raw_page(self, num: int) -> bytes:
# Check if the latest valid instance of the page is committed (either the frame itself
# is the commit frame or it is included in a commit's frames). If so, return that frame's data.
for commit in reversed(self.wal.commits):
- if (frame := commit.get(num)) and frame.valid(validate_checksum=self.validate_checksum):
+ if (frame := commit.get(num)) and frame.is_valid(validate_checksum=self.validate_checksum):
data = frame.data
break
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index baa5dba..0204ebe 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -125,8 +125,8 @@ def __init__(self, wal: WAL, offset: int):
def __repr__(self) -> str:
return f""
- def valid(self, validate_checksum: bool = True) -> bool:
- """Check if the frame is valid by comparing its salt values and optionally verifying the checksum.
+ def is_valid(self, validate_checksum: bool = True) -> bool:
+ """Return whether the frame is valid by comparing its salt values and optionally verifying the checksum.
A frame is valid if:
- Its salt1 and salt2 values match those in the WAL header.
@@ -135,7 +135,7 @@ def valid(self, validate_checksum: bool = True) -> bool:
References:
- https://sqlite.org/fileformat2.html#wal_file_format
"""
- return self.validate_salt() and self.validate_checksum() if validate_checksum else self.validate_salt()
+ return self.is_valid_salt() and self.is_valid_checksum() if validate_checksum else self.is_valid_salt()
def is_valid_salt(self) -> bool:
"""Return whether the frame's salt values match those in the WAL header.
From 64470ddaceea239740a22b7c44fcc68944ba720a Mon Sep 17 00:00:00 2001
From: Pim <36573021+PimSanders@users.noreply.github.com>
Date: Thu, 19 Feb 2026 07:33:49 +0100
Subject: [PATCH 04/19] Apply suggestions from code review
Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com>
---
dissect/database/sqlite3/sqlite3.py | 3 ++-
dissect/database/sqlite3/wal.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py
index e8c171f..fa6d0fe 100644
--- a/dissect/database/sqlite3/sqlite3.py
+++ b/dissect/database/sqlite3/sqlite3.py
@@ -80,7 +80,8 @@ def __init__(
fh: Path | BinaryIO,
wal: WAL | Path | BinaryIO | None = None,
checkpoint: Checkpoint | int | None = None,
- validate_checksum: bool = False,
+ *,
+ validate_checksums: bool = False,
):
if isinstance(fh, Path):
path = fh
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 0204ebe..3d9b9f1 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -135,7 +135,7 @@ def is_valid(self, validate_checksum: bool = True) -> bool:
References:
- https://sqlite.org/fileformat2.html#wal_file_format
"""
- return self.is_valid_salt() and self.is_valid_checksum() if validate_checksum else self.is_valid_salt()
+ return (self.is_valid_salt() and self.is_valid_checksum()) if validate_checksum else self.is_valid_salt()
def is_valid_salt(self) -> bool:
"""Return whether the frame's salt values match those in the WAL header.
From 4572811991a2d7a835c5ae190b4abc680f40f672 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Thu, 19 Feb 2026 07:38:19 +0100
Subject: [PATCH 05/19] Apply suggestions from code review
---
dissect/database/sqlite3/sqlite3.py | 6 +--
dissect/database/sqlite3/wal.py | 78 ++++++++++++-----------------
tests/sqlite3/test_wal.py | 8 +--
3 files changed, 38 insertions(+), 54 deletions(-)
diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py
index fa6d0fe..1528ba7 100644
--- a/dissect/database/sqlite3/sqlite3.py
+++ b/dissect/database/sqlite3/sqlite3.py
@@ -66,7 +66,7 @@ class SQLite3:
fh: The path or file-like object to open a SQLite3 database on.
wal: The path or file-like object to open a SQLite3 WAL file on.
checkpoint: The checkpoint to apply from the WAL file. Can be a :class:`Checkpoint` object or an integer index.
- validate_checksum: A boolean that sets whether to validate the checksum of frames when reading.
+ validate_checksums: A boolean that sets whether to validate the checksum of frames when reading.
Raises:
InvalidDatabase: If the file-like object does not look like a SQLite3 database based on the header magic.
@@ -93,7 +93,7 @@ def __init__(
self.path = path
self.wal = None
self.checkpoint = None
- self.validate_checksum = validate_checksum
+ self.validate_checksums = validate_checksums
self.header = c_sqlite3.header(self.fh)
if self.header.magic != SQLITE3_HEADER_MAGIC:
@@ -215,7 +215,7 @@ def raw_page(self, num: int) -> bytes:
# Check if the latest valid instance of the page is committed (either the frame itself
# is the commit frame or it is included in a commit's frames). If so, return that frame's data.
for commit in reversed(self.wal.commits):
- if (frame := commit.get(num)) and frame.is_valid(validate_checksum=self.validate_checksum):
+ if (frame := commit.get(num)) and frame.is_valid(validate_checksums=self.validate_checksums):
data = frame.data
break
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 3d9b9f1..87a982a 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -125,7 +125,7 @@ def __init__(self, wal: WAL, offset: int):
def __repr__(self) -> str:
return f""
- def is_valid(self, validate_checksum: bool = True) -> bool:
+ def is_valid(self, validate_checksums: bool = True) -> bool:
"""Return whether the frame is valid by comparing its salt values and optionally verifying the checksum.
A frame is valid if:
@@ -135,7 +135,7 @@ def is_valid(self, validate_checksum: bool = True) -> bool:
References:
- https://sqlite.org/fileformat2.html#wal_file_format
"""
- return (self.is_valid_salt() and self.is_valid_checksum()) if validate_checksum else self.is_valid_salt()
+ return (self.is_valid_salt() and self.is_valid_checksum()) if validate_checksums else self.is_valid_salt()
def is_valid_salt(self) -> bool:
"""Return whether the frame's salt values match those in the WAL header.
@@ -163,51 +163,35 @@ def is_valid_checksum(self) -> bool:
- https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
"""
checksum_match = False
- base_position = self.fh.tell()
- try:
- # Read the WAL header bytes from the beginning of the file
- wal_hdr_size = len(c_sqlite3.wal_header)
- wal_hdr_bytes = self.wal.header.dumps()
- if len(wal_hdr_bytes) < wal_hdr_size:
- raise EOFError("WAL header too small for checksum calculation")
-
- # Start seed with checksum over first 24 bytes of WAL header
- seed = calculate_checksum(wal_hdr_bytes[:24], endian=self.wal.checksum_endian)
-
- # Iterate frames from the first frame up to and including this frame
- frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size
- first_frame_offset = len(c_sqlite3.wal_header)
- offset = first_frame_offset
-
- while offset <= self.offset:
- # Read frame header
- self.fh.seek(offset)
- frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame))
- if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame):
- raise EOFError("Incomplete frame header while calculating checksum")
-
- # Checksum first 16 bytes of frame header
- seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian)
-
- # Read and checksum page data
- page_offset = offset + len(c_sqlite3.wal_frame)
- self.fh.seek(page_offset)
- page_data = self.fh.read(self.wal.header.page_size)
- if len(page_data) < self.wal.header.page_size:
- raise EOFError("Incomplete page data while calculating checksum")
- seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian)
-
- offset += frame_size
-
- # Compare calculated checksum to stored checksum in this frame header
- checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2)
-
- finally:
- # restore file position
- try:
- self.fh.seek(base_position)
- except Exception:
- pass
+
+ # Start seed with checksum over first 24 bytes of WAL header
+ seed = calculate_checksum(self.header.dumps()[:24], endian=self.wal.checksum_endian)
+
+ # Iterate frames from the first frame up to and including this frame
+ frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size
+ first_frame_offset = len(c_sqlite3.wal_header)
+ offset = first_frame_offset
+
+ while offset <= self.offset:
+ # Read frame header
+ self.fh.seek(offset)
+ frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame))
+ if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame):
+ raise EOFError("Incomplete frame header while calculating checksum")
+
+ # Checksum first 16 bytes of frame header
+ seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian)
+
+ # Read and checksum page data
+ page_data = self.fh.read(self.wal.header.page_size)
+ if len(page_data) < self.wal.header.page_size:
+ raise EOFError("Incomplete page data while calculating checksum")
+ seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian)
+
+ offset += frame_size
+
+ # Compare calculated checksum to stored checksum in this frame header
+ checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2)
return checksum_match
diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py
index 57ca74c..46fe0b3 100644
--- a/tests/sqlite3/test_wal.py
+++ b/tests/sqlite3/test_wal.py
@@ -57,22 +57,22 @@ def test_sqlite_wal_checkpoint(sqlite_db: Path, sqlite_wal: Path, db_as_path: bo
)
def test_sqlite_wal_checksum_validation(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None:
# Test that the WAL checksum validation works as expected
- # When validate_checksum=True, only entries before the last checkpoint are visible
+ # When validate_checksums=True, only entries before the last checkpoint are visible
db = sqlite3.SQLite3(
sqlite_db if db_as_path else sqlite_db.open("rb"),
sqlite_wal if wal_as_path else sqlite_wal.open("rb"),
- validate_checksum=True,
+ validate_checksums=True,
)
_assert_valid_checksum(db)
db.close()
- # When validate_checksum=False, entries after the last checkpoint are also visible
+ # When validate_checksums=False, entries after the last checkpoint are also visible
db = sqlite3.SQLite3(
sqlite_db if db_as_path else sqlite_db.open("rb"),
sqlite_wal if wal_as_path else sqlite_wal.open("rb"),
- validate_checksum=False,
+ validate_checksums=False,
)
_assert_invalid_checksum(db)
From f4b6ffb07c113f876b1e91c84d8982545e9cae66 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Thu, 19 Feb 2026 07:42:53 +0100
Subject: [PATCH 06/19] Fix linter
---
dissect/database/sqlite3/wal.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 87a982a..b40799f 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -162,8 +162,6 @@ def is_valid_checksum(self) -> bool:
- https://sqlite.org/fileformat2.html#wal_file_format
- https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
"""
- checksum_match = False
-
# Start seed with checksum over first 24 bytes of WAL header
seed = calculate_checksum(self.header.dumps()[:24], endian=self.wal.checksum_endian)
@@ -191,9 +189,7 @@ def is_valid_checksum(self) -> bool:
offset += frame_size
# Compare calculated checksum to stored checksum in this frame header
- checksum_match = (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2)
-
- return checksum_match
+ return (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2)
@property
def data(self) -> bytes:
From 20ffbfb5a67b93709641bc9cd334719f091b743e Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Wed, 27 May 2026 21:03:57 +0200
Subject: [PATCH 07/19] Add benchmark
---
dissect/database/sqlite3/wal.py | 5 +++--
tests/_data/sqlite3/big.sqlite | 3 +++
tests/_data/sqlite3/big.sqlite-wal | 3 +++
tests/sqlite3/conftest.py | 10 ++++++++++
tests/sqlite3/test_wal.py | 18 ++++++++++++++++++
5 files changed, 37 insertions(+), 2 deletions(-)
create mode 100644 tests/_data/sqlite3/big.sqlite
create mode 100644 tests/_data/sqlite3/big.sqlite-wal
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index c37dbc3..2261fe8 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -39,7 +39,9 @@ def __init__(self, fh: Path | BinaryIO):
raise InvalidDatabase("Invalid WAL header magic")
self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">"
- self.highest_page_num = max(fr.page_number for commit in self.commits for fr in commit.frames if fr.valid)
+ self.highest_page_num = max(
+ fr.page_number for commit in self.commits for fr in commit.frames if fr.is_valid_salt()
+ )
self.frame = lru_cache(1024)(self.frame)
@@ -253,7 +255,6 @@ def calculate_checksum(buf: bytes, seed: tuple[int, int] = (0, 0), endian: str =
References:
- https://sqlite.org/fileformat2.html#checksum_algorithm
"""
-
s0, s1 = seed
num_ints = len(buf) // 4
arr = struct.unpack(f"{endian}{num_ints}I", buf)
diff --git a/tests/_data/sqlite3/big.sqlite b/tests/_data/sqlite3/big.sqlite
new file mode 100644
index 0000000..bb2cb0f
--- /dev/null
+++ b/tests/_data/sqlite3/big.sqlite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49abb18da561d667cd02cd3f4aa5ad6f80740f29f9c2a24b6e7286cbe259ffe2
+size 69632
diff --git a/tests/_data/sqlite3/big.sqlite-wal b/tests/_data/sqlite3/big.sqlite-wal
new file mode 100644
index 0000000..2df6fe5
--- /dev/null
+++ b/tests/_data/sqlite3/big.sqlite-wal
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ada96856785eed8f8aca40d43f788cdb515eed6fcd40b77004134471c9160fd8
+size 8305952
diff --git a/tests/sqlite3/conftest.py b/tests/sqlite3/conftest.py
index 9f86947..3579c0f 100644
--- a/tests/sqlite3/conftest.py
+++ b/tests/sqlite3/conftest.py
@@ -23,3 +23,13 @@ def sqlite_wal() -> Path:
@pytest.fixture
def empty_db() -> Path:
return absolute_path("_data/sqlite3/empty.sqlite")
+
+
+@pytest.fixture
+def big_sqlite_db() -> Path:
+ return absolute_path("_data/sqlite3/big.sqlite")
+
+
+@pytest.fixture
+def big_sqlite_wal() -> Path:
+ return absolute_path("_data/sqlite3/big.sqlite-wal")
diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py
index 5e3b281..d7f91b9 100644
--- a/tests/sqlite3/test_wal.py
+++ b/tests/sqlite3/test_wal.py
@@ -10,6 +10,8 @@
if TYPE_CHECKING:
from pathlib import Path
+ from pytest_benchmark.fixture import BenchmarkFixture
+
@pytest.mark.parametrize(
("db_as_path"),
@@ -314,3 +316,19 @@ def test_wal_page_count() -> None:
assert db.wal.highest_page_num == 4
assert db.header.page_count == 2
assert db.page_count == 4
+
+
+@pytest.mark.parametrize(
+ ("validate"),
+ [pytest.param(True, id="True"), pytest.param(False, id="False")],
+)
+@pytest.mark.benchmark
+def test_benchmark_wal_checksum_validation(
+ big_sqlite_db: Path, big_sqlite_wal: Path, validate: bool, benchmark: BenchmarkFixture
+) -> None:
+ def benchy() -> None:
+ # list(db.tables()[0].rows())
+ db = sqlite3.SQLite3(big_sqlite_db, big_sqlite_wal, validate_checksums=validate)
+ list(list(db.tables())[0].rows())
+
+ benchmark(benchy)
From 5d0cbc732e8ba9f306167c91ab8ccd4ffdbefe43 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Wed, 27 May 2026 21:07:00 +0200
Subject: [PATCH 08/19] Fix linter again, as usual
---
tests/sqlite3/test_wal.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py
index d7f91b9..5db5fac 100644
--- a/tests/sqlite3/test_wal.py
+++ b/tests/sqlite3/test_wal.py
@@ -329,6 +329,6 @@ def test_benchmark_wal_checksum_validation(
def benchy() -> None:
# list(db.tables()[0].rows())
db = sqlite3.SQLite3(big_sqlite_db, big_sqlite_wal, validate_checksums=validate)
- list(list(db.tables())[0].rows())
+ list(next(iter(db.tables())))
benchmark(benchy)
From c70d440543adb9904787376d10b88e90072996ab Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Fri, 29 May 2026 21:56:48 +0200
Subject: [PATCH 09/19] Remove commented line
---
tests/sqlite3/test_wal.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py
index 5db5fac..d3ffdd5 100644
--- a/tests/sqlite3/test_wal.py
+++ b/tests/sqlite3/test_wal.py
@@ -327,7 +327,6 @@ def test_benchmark_wal_checksum_validation(
big_sqlite_db: Path, big_sqlite_wal: Path, validate: bool, benchmark: BenchmarkFixture
) -> None:
def benchy() -> None:
- # list(db.tables()[0].rows())
db = sqlite3.SQLite3(big_sqlite_db, big_sqlite_wal, validate_checksums=validate)
list(next(iter(db.tables())))
From ba48643a61be84a1c593dd6e0ac40953b5df649b Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Fri, 29 May 2026 22:16:09 +0200
Subject: [PATCH 10/19] Cache initial checksum from WAL header
---
dissect/database/sqlite3/wal.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 2261fe8..6f4f276 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -114,6 +114,11 @@ def checkpoints(self) -> list[Checkpoint]:
return [checkpoints_map[salt] for salt in sorted(checkpoints_map.keys())]
+ @cached_property
+ def header_checksum_seed(self) -> tuple[int, int]:
+ """Cached initial checksum seed calculated from the WAL header first 24 bytes."""
+ return calculate_checksum(self.header.dumps()[:24], endian=self.checksum_endian)
+
class Frame:
def __init__(self, wal: WAL, offset: int):
@@ -165,8 +170,8 @@ def is_valid_checksum(self) -> bool:
- https://sqlite.org/fileformat2.html#wal_file_format
- https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
"""
- # Start seed with checksum over first 24 bytes of WAL header
- seed = calculate_checksum(self.header.dumps()[:24], endian=self.wal.checksum_endian)
+ # Start seed with checksum over first 24 bytes of WAL header (cached on WAL)
+ seed = self.wal.header_checksum_seed
# Iterate frames from the first frame up to and including this frame
frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size
From a335ccc9a6dc124ceee46f3698fd8b931d1af711 Mon Sep 17 00:00:00 2001
From: Pim <36573021+PimSanders@users.noreply.github.com>
Date: Fri, 29 May 2026 22:18:12 +0200
Subject: [PATCH 11/19] Update dissect/database/sqlite3/wal.py
Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com>
---
dissect/database/sqlite3/wal.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 6f4f276..c289cc8 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -163,7 +163,7 @@ def is_valid_checksum(self) -> bool:
exactly match the computed checksum over:
1. the first 24 bytes of the WAL header
- 2. the first 8 bytes of each frame header (up to and including this frame)
+ 2. the first 16 bytes of each frame header (up to and including this frame)
3. the page data of each frame (up to and including this frame)
References:
From 7400171d6f73ae11b468d946fb94c7d0cb5783fb Mon Sep 17 00:00:00 2001
From: Pim <36573021+PimSanders@users.noreply.github.com>
Date: Sun, 21 Jun 2026 15:28:28 +0200
Subject: [PATCH 12/19] Use last seed method
---
dissect/database/sqlite3/wal.py | 119 +++++++++++++++++++++-----------
1 file changed, 78 insertions(+), 41 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index c289cc8..2ce3a65 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -39,12 +39,24 @@ def __init__(self, fh: Path | BinaryIO):
raise InvalidDatabase("Invalid WAL header magic")
self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">"
+
+ self.frame = lru_cache(1024)(self.frame)
+ self.frame_size = len(c_sqlite3.wal_frame) + self.header.page_size
+ self.first_frame_offset = len(c_sqlite3.wal_header)
+
+ # Only track the highest valid offset and its seed.
+ # Meaning: all frames with offset < _highest_valid_next_offset are considered valid.
+ # _highest_valid_next_offset initially points at the first frame; seed is checksum over header.
+ self._highest_valid_next_offset: int = self.first_frame_offset
+ self._highest_valid_seed: tuple[int, int] = self.header_checksum_seed
+
+ # First offset that is known to fail checksum validation, or None.
+ self._checksum_failed_offset: int | None = None
+
self.highest_page_num = max(
fr.page_number for commit in self.commits for fr in commit.frames if fr.is_valid_salt()
)
- self.frame = lru_cache(1024)(self.frame)
-
def close(self) -> None:
"""Close the WAL."""
# Only close WAL handle if we opened it using a path
@@ -52,8 +64,7 @@ def close(self) -> None:
self.fh.close()
def frame(self, frame_idx: int) -> Frame:
- frame_size = len(c_sqlite3.wal_frame) + self.header.page_size
- offset = len(c_sqlite3.wal_header) + frame_idx * frame_size
+ offset = self.first_frame_offset + frame_idx * self.frame_size
return Frame(self, offset)
def frames(self) -> Iterator[Frame]:
@@ -65,6 +76,62 @@ def frames(self) -> Iterator[Frame]:
except EOFError: # noqa: PERF203
break
+ def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None:
+ """Return checksum seed after processing frames up to and including the frame at target_offset.
+
+ If validate=True, verify stored checksums for each frame as we walk. If a mismatch is found, update
+ the WAL's highest-known-valid-next-offset and return None. On success (no mismatches) update
+ the highest-known-valid-next-offset and seed and return the computed seed.
+
+ References:
+ - https://sqlite.org/fileformat2.html#wal_file_format
+ - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
+ """
+ # If the target offset is before the first frame, return the initial seed calculated from the WAL header.
+ if target_offset < self.first_frame_offset:
+ return self.header_checksum_seed
+
+ # Start from the highest verified offset we know (saves re-checking earlier frames).
+ base_offset = (
+ self._highest_valid_next_offset
+ if self._highest_valid_next_offset <= target_offset
+ else self.first_frame_offset
+ )
+ seed = self._highest_valid_seed if base_offset == self._highest_valid_next_offset else self.header_checksum_seed
+ offset = base_offset
+
+ while offset <= target_offset:
+ # Read frame header
+ self.fh.seek(offset)
+ frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame))
+ if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame):
+ raise EOFError("Incomplete frame header while calculating checksum")
+
+ # Checksum first 16 bytes of frame header
+ seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.checksum_endian)
+
+ # Read and checksum page data
+ page_data = self.fh.read(self.header.page_size)
+ if len(page_data) < self.header.page_size:
+ raise EOFError("Incomplete page data while calculating checksum")
+ seed = calculate_checksum(page_data, seed=seed, endian=self.checksum_endian)
+
+ # Compare computed seed to stored checksums in this frame header.
+ checksum1, checksum2 = struct.unpack(f"{self.checksum_endian}2I", frame_hdr_bytes[-8:])
+ if (seed[0], seed[1]) != (checksum1, checksum2):
+ self._highest_valid_next_offset = min(self._highest_valid_next_offset, offset)
+ self._checksum_failed_offset = offset
+
+ return None
+
+ offset += self.frame_size
+
+ # Update highest-known-valid-next-offset and seed to the next offset after target.
+ self._highest_valid_next_offset = offset
+ self._highest_valid_seed = seed
+
+ return seed
+
@cached_property
def commits(self) -> list[Commit]:
"""Return all commits in the WAL file.
@@ -159,45 +226,15 @@ def is_valid_salt(self) -> bool:
def is_valid_checksum(self) -> bool:
"""Return whether the frame's checksum matches the calculated checksum.
- The checksum values in the final 8 bytes of the frame-header (checksum-1 and checksum-2)
- exactly match the computed checksum over:
-
- 1. the first 24 bytes of the WAL header
- 2. the first 16 bytes of each frame header (up to and including this frame)
- 3. the page data of each frame (up to and including this frame)
-
- References:
- - https://sqlite.org/fileformat2.html#wal_file_format
- - https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
+ Use WAL's highest valid offset to skip checks for already-verified frames.
"""
- # Start seed with checksum over first 24 bytes of WAL header (cached on WAL)
- seed = self.wal.header_checksum_seed
-
- # Iterate frames from the first frame up to and including this frame
- frame_size = len(c_sqlite3.wal_frame) + self.wal.header.page_size
- first_frame_offset = len(c_sqlite3.wal_header)
- offset = first_frame_offset
-
- while offset <= self.offset:
- # Read frame header
- self.fh.seek(offset)
- frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame))
- if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame):
- raise EOFError("Incomplete frame header while calculating checksum")
-
- # Checksum first 16 bytes of frame header
- seed = calculate_checksum(frame_hdr_bytes[:16], seed=seed, endian=self.wal.checksum_endian)
-
- # Read and checksum page data
- page_data = self.fh.read(self.wal.header.page_size)
- if len(page_data) < self.wal.header.page_size:
- raise EOFError("Incomplete page data while calculating checksum")
- seed = calculate_checksum(page_data, seed=seed, endian=self.wal.checksum_endian)
-
- offset += frame_size
+ if self.offset < self.wal._highest_valid_next_offset:
+ return True
+ if self.wal._checksum_failed_offset is not None and self.offset >= self.wal._checksum_failed_offset:
+ return False
- # Compare calculated checksum to stored checksum in this frame header
- return (seed[0], seed[1]) == (self.header.checksum1, self.header.checksum2)
+ seed = self.wal.seed_for_offset(self.offset)
+ return seed is not None
@property
def data(self) -> bytes:
From 92a6b9cd4ef01193ce77fb63c71a4e41421375a6 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Thu, 25 Jun 2026 17:40:29 +0200
Subject: [PATCH 13/19] Rename vars
---
dissect/database/sqlite3/wal.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 2ce3a65..aa92f17 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -76,7 +76,7 @@ def frames(self) -> Iterator[Frame]:
except EOFError: # noqa: PERF203
break
- def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None:
+ def seed_for_offset(self, offset: int) -> tuple[int, int] | None:
"""Return checksum seed after processing frames up to and including the frame at target_offset.
If validate=True, verify stored checksums for each frame as we walk. If a mismatch is found, update
@@ -88,21 +88,21 @@ def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None:
- https://github.com/sqlite/sqlite/blob/master/src/wal.c#L995-L1047
"""
# If the target offset is before the first frame, return the initial seed calculated from the WAL header.
- if target_offset < self.first_frame_offset:
+ if offset < self.first_frame_offset:
return self.header_checksum_seed
# Start from the highest verified offset we know (saves re-checking earlier frames).
base_offset = (
self._highest_valid_next_offset
- if self._highest_valid_next_offset <= target_offset
+ if self._highest_valid_next_offset <= offset
else self.first_frame_offset
)
seed = self._highest_valid_seed if base_offset == self._highest_valid_next_offset else self.header_checksum_seed
- offset = base_offset
+ current_offset = base_offset
- while offset <= target_offset:
+ while current_offset <= offset:
# Read frame header
- self.fh.seek(offset)
+ self.fh.seek(current_offset)
frame_hdr_bytes = self.fh.read(len(c_sqlite3.wal_frame))
if len(frame_hdr_bytes) < len(c_sqlite3.wal_frame):
raise EOFError("Incomplete frame header while calculating checksum")
@@ -119,15 +119,15 @@ def seed_for_offset(self, target_offset: int) -> tuple[int, int] | None:
# Compare computed seed to stored checksums in this frame header.
checksum1, checksum2 = struct.unpack(f"{self.checksum_endian}2I", frame_hdr_bytes[-8:])
if (seed[0], seed[1]) != (checksum1, checksum2):
- self._highest_valid_next_offset = min(self._highest_valid_next_offset, offset)
- self._checksum_failed_offset = offset
+ self._highest_valid_next_offset = min(self._highest_valid_next_offset, current_offset)
+ self._checksum_failed_offset = current_offset
return None
- offset += self.frame_size
+ current_offset += self.frame_size
# Update highest-known-valid-next-offset and seed to the next offset after target.
- self._highest_valid_next_offset = offset
+ self._highest_valid_next_offset = current_offset
self._highest_valid_seed = seed
return seed
From 0b72b703941e833318381719111a33b56b0c262b Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Thu, 25 Jun 2026 17:43:28 +0200
Subject: [PATCH 14/19] Update comment
---
dissect/database/sqlite3/wal.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index aa92f17..2a26fa0 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -79,9 +79,9 @@ def frames(self) -> Iterator[Frame]:
def seed_for_offset(self, offset: int) -> tuple[int, int] | None:
"""Return checksum seed after processing frames up to and including the frame at target_offset.
- If validate=True, verify stored checksums for each frame as we walk. If a mismatch is found, update
- the WAL's highest-known-valid-next-offset and return None. On success (no mismatches) update
- the highest-known-valid-next-offset and seed and return the computed seed.
+ Verify stored checksums for each frame as we walk. If a mismatch is found, update the WAL's
+ highest-known-valid-next-offset and return None. On success (no mismatches) update the
+ highest-known-valid-next-offset and seed and return the computed seed.
References:
- https://sqlite.org/fileformat2.html#wal_file_format
From d0f5d4fb1dce1b37d56955b8928650ce2e4b8a6f Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Thu, 25 Jun 2026 17:51:07 +0200
Subject: [PATCH 15/19] Remove redundant if statemnets
---
dissect/database/sqlite3/wal.py | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 2a26fa0..5ea87a5 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -92,13 +92,8 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None:
return self.header_checksum_seed
# Start from the highest verified offset we know (saves re-checking earlier frames).
- base_offset = (
- self._highest_valid_next_offset
- if self._highest_valid_next_offset <= offset
- else self.first_frame_offset
- )
- seed = self._highest_valid_seed if base_offset == self._highest_valid_next_offset else self.header_checksum_seed
- current_offset = base_offset
+ current_offset = self._highest_valid_next_offset
+ seed = self._highest_valid_seed
while current_offset <= offset:
# Read frame header
From 61758d90da1b1fa1a306c8f2c59910bd6e10f1ea Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Tue, 7 Jul 2026 17:56:25 +0200
Subject: [PATCH 16/19] Use precompiled struct
---
dissect/database/sqlite3/wal.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 5ea87a5..bc59520 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -39,6 +39,7 @@ def __init__(self, fh: Path | BinaryIO):
raise InvalidDatabase("Invalid WAL header magic")
self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">"
+ self._checksum_struct = struct.Struct(f"{self.checksum_endian}2I")
self.frame = lru_cache(1024)(self.frame)
self.frame_size = len(c_sqlite3.wal_frame) + self.header.page_size
@@ -112,7 +113,7 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None:
seed = calculate_checksum(page_data, seed=seed, endian=self.checksum_endian)
# Compare computed seed to stored checksums in this frame header.
- checksum1, checksum2 = struct.unpack(f"{self.checksum_endian}2I", frame_hdr_bytes[-8:])
+ checksum1, checksum2 = self._checksum_struct.unpack(frame_hdr_bytes[-8:])
if (seed[0], seed[1]) != (checksum1, checksum2):
self._highest_valid_next_offset = min(self._highest_valid_next_offset, current_offset)
self._checksum_failed_offset = current_offset
From 9bd3e762670ab803aa489934f35caf6a5d6960e1 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Tue, 7 Jul 2026 17:59:39 +0200
Subject: [PATCH 17/19] Do not update self._highest_valid_next_offset when
checksum fails
---
dissect/database/sqlite3/wal.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index bc59520..9a1650b 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -115,9 +115,7 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None:
# Compare computed seed to stored checksums in this frame header.
checksum1, checksum2 = self._checksum_struct.unpack(frame_hdr_bytes[-8:])
if (seed[0], seed[1]) != (checksum1, checksum2):
- self._highest_valid_next_offset = min(self._highest_valid_next_offset, current_offset)
self._checksum_failed_offset = current_offset
-
return None
current_offset += self.frame_size
From 114bdc20355b785be2959043ca29a5c6f78e5c45 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Fri, 10 Jul 2026 14:59:51 +0200
Subject: [PATCH 18/19] Move failed offset check to seed_for_offset
---
dissect/database/sqlite3/wal.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py
index 9a1650b..04a538a 100644
--- a/dissect/database/sqlite3/wal.py
+++ b/dissect/database/sqlite3/wal.py
@@ -92,6 +92,10 @@ def seed_for_offset(self, offset: int) -> tuple[int, int] | None:
if offset < self.first_frame_offset:
return self.header_checksum_seed
+ # If the target offset is at or beyond the first known checksum failure, return None.
+ if self._checksum_failed_offset is not None and offset >= self._checksum_failed_offset:
+ return None
+
# Start from the highest verified offset we know (saves re-checking earlier frames).
current_offset = self._highest_valid_next_offset
seed = self._highest_valid_seed
@@ -224,8 +228,6 @@ def is_valid_checksum(self) -> bool:
"""
if self.offset < self.wal._highest_valid_next_offset:
return True
- if self.wal._checksum_failed_offset is not None and self.offset >= self.wal._checksum_failed_offset:
- return False
seed = self.wal.seed_for_offset(self.offset)
return seed is not None
From ec7b77e169117364b92ef64068c0dcacb39d41f8 Mon Sep 17 00:00:00 2001
From: Pim Sanders <36573021+PimSanders@users.noreply.github.com>
Date: Fri, 10 Jul 2026 22:30:42 +0200
Subject: [PATCH 19/19] Enable checksum validation by default
---
dissect/database/sqlite3/sqlite3.py | 2 +-
tests/sqlite3/test_sqlite3.py | 4 ++--
tests/sqlite3/test_wal.py | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py
index 2613ce3..7fdabcd 100644
--- a/dissect/database/sqlite3/sqlite3.py
+++ b/dissect/database/sqlite3/sqlite3.py
@@ -84,7 +84,7 @@ def __init__(
wal: WAL | Path | BinaryIO | None = None,
checkpoint: Checkpoint | int | None = None,
*,
- validate_checksums: bool = False,
+ validate_checksums: bool = True,
):
if isinstance(fh, Path):
path = fh
diff --git a/tests/sqlite3/test_sqlite3.py b/tests/sqlite3/test_sqlite3.py
index f5caaf9..e56a26b 100644
--- a/tests/sqlite3/test_sqlite3.py
+++ b/tests/sqlite3/test_sqlite3.py
@@ -17,11 +17,11 @@
[pytest.param(True, id="as_path"), pytest.param(False, id="as_fh")],
)
def test_sqlite(sqlite_db: Path, open_as_path: bool) -> None:
- db = sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb"))
+ db = sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb"), validate_checksums=False)
_assert_sqlite_db(db)
db.close()
- with sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb")) as db:
+ with sqlite3.SQLite3(sqlite_db if open_as_path else sqlite_db.open("rb"), validate_checksums=False) as db:
_assert_sqlite_db(db)
diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py
index d3ffdd5..8c71f71 100644
--- a/tests/sqlite3/test_wal.py
+++ b/tests/sqlite3/test_wal.py
@@ -304,7 +304,7 @@ def test_wal_page_count() -> None:
>>> con.commit()
# Copy page_count.db* files before closing
"""
- db = sqlite3.SQLite3(absolute_path("_data/sqlite3/page_count.db"))
+ db = sqlite3.SQLite3(absolute_path("_data/sqlite3/page_count.db"), validate_checksums=False)
table = db.table("t1")
assert table.sql == "CREATE TABLE t1 (a, b)"