From 382c98c4dc9be72682740b465ed0715ca7ca6cfb Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 19 Aug 2026 13:49:26 +0100 Subject: [PATCH 1/5] Return message upon completion of the registration function --- src/murfey/workflows/fib/register_lamella_evaluation_image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index 2e826f16a..07b785647 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -15,3 +15,4 @@ def run( "Received message containing the following:\n" f"{json.dumps(message, indent=2, default=str)}" ) + return {"success": True} From b6803acce21ca3349becfa730179044272d1e851 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 19 Aug 2026 17:24:16 +0100 Subject: [PATCH 2/5] Updated workflow to extract metadata from the lamella evaluation image --- .../fib/register_lamella_evaluation_image.py | 203 +++++++++++++++++- 1 file changed, 195 insertions(+), 8 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index 07b785647..d1dd0d466 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -1,18 +1,205 @@ -import json import logging -from typing import Any +import math +import xml.etree.ElementTree as ET +from functools import cached_property +from pathlib import Path +from typing import Any, cast -from sqlmodel import Session +import PIL.Image +from pydantic import BaseModel, computed_field, model_validator +from sqlmodel import Session, select + +import murfey.util.db as MurfeyDB +from murfey.util.config import get_machine_config +from murfey.util.fib import get_slot_number logger = logging.getLogger(__name__) +class FIBImageMetadata(BaseModel): + """ + These fields should ALL be present in the Electron Snapshot image. + Positions and pixel sizes are in metres, whereas angles are in radians. + """ + + visit_name: str + file: Path + thumbnail_path: Path | None = None + # Acceleration voltage + voltage: float + # Beam shifts + shift_x: float + shift_y: float + # Actual field of view + len_x: float + len_y: float + # Stage position + pos_x: float + pos_y: float + pos_z: float + rotation: float # Radians + slot_number: int + tilt_alpha: float # Radians + tilt_beta: float # Radians + # Image dimensions + pixels_x: int + pixels_y: int + # Pixel size + pixel_size_x: float + pixel_size_y: float + + @model_validator(mode="after") + def check_pixel_size_tolerance(self): + """ + The pixel size values for x and y should be nigh-identical + """ + if abs(self.pixel_size_x - self.pixel_size_y) > 1e-18: + raise ValueError + return self + + # mypy doesn't support decorators on @property + @computed_field # type: ignore + @cached_property + def pixel_size(self) -> float: + """ + Return an average of pixel sizes along the x- and y-axes + """ + return 0.5 * (self.pixel_size_x + self.pixel_size_y) + + # mypy doesn't support decorators on @property + @computed_field # type: ignore + @cached_property + def project_name(self) -> str: + """ + Extract the project name from the file path. This assumes a specific + folder structure of '{visit_name}/maps/{project_name}'. + """ + path_parts = self.file.parts + visit_idx = path_parts.index(self.visit_name) + return path_parts[visit_idx + 2] # {visit}/maps/{project_name} + + # mypy doesn't support decorators on @property + @computed_field # type: ignore + @cached_property + def site_name(self) -> str: + """ + Create a site name for the current image based on the project name + and its slot number. + """ + return f"{self.project_name}--slot_{self.slot_number}" + + +def _parse_metadata(file: Path, visit_name: str, rotation_offset: float): + """ + Parses through the atlas image's tags to extract the relevant metadata + """ + + # Search for the XML metadata in the tags (34683 is the default key) + img = PIL.Image.open(file) + tags = dict(img.text) + xml_metadata = None + if ( + isinstance((tag_contents := tags.get("Metadata")), str) + and "xml version" in tag_contents + ): + xml_metadata = ET.fromstring(tag_contents) + if xml_metadata is None: + raise ValueError(f"Could not find required metadata in file {file}") + + # Extract key values from metadata + extracted: dict[str, Any] = { + key: node.text if (node := xml_metadata.find(node_path)) is not None else None + for key, node_path in ( + ("voltage", ".//Optics/AccelerationVoltage"), + ("shift_x", ".//Optics/BeamShift/X"), + ("shift_y", ".//Optics/BeamShift/Y"), + ("len_x", ".//Optics/ScanFieldOfView/X"), + ("len_y", ".//Optics/ScanFieldOfView/Y"), + ("pos_x", ".//StageSettings/StagePosition/X"), + ("pos_y", ".//StageSettings/StagePosition/Y"), + ("pos_z", ".//StageSettings/StagePosition/Z"), + # Angles are in radians + ("rotation", ".//StageSettings/StagePosition/Rotation"), + ("tilt_alpha", ".//StageSettings/StagePosition/Tilt/Alpha"), + ("tilt_beta", ".//StageSettings/StagePosition/Tilt/Beta"), + ("pixels_x", ".//BinaryResult/ImageSize/X"), + ("pixels_y", ".//BinaryResult/ImageSize/Y"), + ("pixel_size_x", ".//BinaryResult/PixelSize/X"), + ("pixel_size_y", ".//BinaryResult/PixelSize/Y"), + ) + } + # Calculate the slot number + extracted["slot_number"] = get_slot_number( + x=float(extracted["pos_x"]), + y=float(extracted["pos_y"]), + rotation=math.degrees(float(extracted["rotation"])), # Convert to degrees + rotation_offset=rotation_offset, + ) + # Return the parsed Pydantic model + return FIBImageMetadata( + visit_name=visit_name, + file=file, + **extracted, + ) + + +class FIBLamellaImageInfo(BaseModel): + session_id: int + lamella_image_file: Path + + def run( message: dict[str, Any], murfey_db: Session, ): - logger.debug( - "Received message containing the following:\n" - f"{json.dumps(message, indent=2, default=str)}" - ) - return {"success": True} + # Outer try-finally block to ensure the database connection is closed + try: + try: + # Validate incoming message + fib_info = FIBLamellaImageInfo(**message) + except Exception: + logger.error("Could not validate incoming message", exc_info=True) + return {"success": False, "requeue": False} + + try: + # Load visit information + murfey_session = murfey_db.exec( + select(MurfeyDB.Session).where( + MurfeyDB.Session.id == fib_info.session_id + ) + ).one() + visit_name = murfey_session.visit + instrument_name = murfey_session.instrument_name + except Exception: + logger.error( + "Exception encountered while querying Murfey database", exc_info=True + ) + return {"success": False, "requeue": False} + + try: + # Load the machine config + machine_config = get_machine_config(instrument_name)[instrument_name] + rotation_offset: float = cast( + float, machine_config.calibrations.get("rotation_offset", 0) + ) + + # Extract metadata from the image + metadata = _parse_metadata( + file=fib_info.lamella_image_file, + visit_name=visit_name, + rotation_offset=rotation_offset, + ) + logger.debug( + "Extracted the following metadata from the image:\n", + metadata.model_dump_json(indent=2), + ) + except Exception: + logger.error( + f"Error extracting metadata from file {fib_info.atlas_file}", + exc_info=True, + ) + return {"success": False, "requeue": False} + + return {"success": True} + finally: + murfey_db.close() From 33586d06af3ec04c4a5693258a5cd1891e240a19 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 19 Aug 2026 17:24:29 +0100 Subject: [PATCH 3/5] Updated tests --- .../test_register_lamella_evaluation_image.py | 365 +++++++++++++++++- 1 file changed, 361 insertions(+), 4 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index 07fbf67a7..e42da6835 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -1,17 +1,374 @@ +import xml.etree.ElementTree as ET +from pathlib import Path from unittest.mock import MagicMock +import pytest from pytest_mock import MockerFixture -from murfey.workflows.fib.register_lamella_evaluation_image import run +from murfey.util.config import MachineConfig +from murfey.workflows.fib.register_lamella_evaluation_image import ( + FIBImageMetadata, + _parse_metadata, + run, +) +from tests.conftest import ExampleVisit + +session_id = 10 +visit_name = f"{ExampleVisit.proposal_code}{ExampleVisit.proposal_number}-{ExampleVisit.visit_number}" +instrument_name = ExampleVisit.instrument_name + + +@pytest.fixture +def visit_dir(tmp_path: Path): + visit_dir = tmp_path / "data/2020" / visit_name + visit_dir.mkdir(parents=True, exist_ok=True) + return visit_dir + + +def create_lamella_evaluation_image_metadata( + voltage: float, + shift_x: float, + shift_y: float, + len_x: float, + len_y: float, + pos_x: float, + pos_y: float, + pos_z: float, + rotation: float, + tilt_alpha: float, + tilt_beta: float, + pixels_x: int, + pixels_y: int, + pixel_size_x: float, + pixel_size_y: float, +): + # Create the XML Element structure present in the file + root = ET.Element("Metadata") + + # ------ + # Optics + # ------ + optics_node = ET.Element("Optics") + + voltage_node = ET.Element("AccelerationVoltage") + voltage_node.text = str(voltage) + optics_node.append(voltage_node) + + beam_shift_node = ET.Element("BeamShift") + shift_x_node = ET.Element("X") + shift_x_node.text = str(shift_x) + beam_shift_node.append(shift_x_node) + shift_y_node = ET.Element("Y") + shift_y_node.text = str(shift_y) + beam_shift_node.append(shift_y_node) + optics_node.append(beam_shift_node) + + fov_node = ET.Element("ScanFieldOfView") + len_x_node = ET.Element("X") + len_x_node.text = str(len_x) + fov_node.append(len_x_node) + len_y_node = ET.Element("Y") + len_y_node.text = str(len_y) + fov_node.append(len_y_node) + optics_node.append(fov_node) + + root.append(optics_node) + + # ------------- + # StageSettings + # ------------- + stage_settings_node = ET.Element("StageSettings") + # x, y, z + stage_node = ET.Element("StagePosition") + pos_x_node = ET.Element("X") + pos_x_node.text = str(pos_x) + stage_node.append(pos_x_node) + pos_y_node = ET.Element("Y") + pos_y_node.text = str(pos_y) + stage_node.append(pos_y_node) + pos_z_node = ET.Element("Z") + pos_z_node.text = str(pos_z) + stage_node.append(pos_z_node) + rotation_node = ET.Element("Rotation") + rotation_node.text = str(rotation) + stage_node.append(rotation_node) + # Angles + tilt_node = ET.Element("Tilt") + tilt_alpha_node = ET.Element("Alpha") + tilt_alpha_node.text = str(tilt_alpha) + tilt_node.append(tilt_alpha_node) + tilt_beta_node = ET.Element("Beta") + tilt_beta_node.text = str(tilt_beta) + tilt_node.append(tilt_beta_node) + stage_node.append(tilt_node) + + stage_settings_node.append(stage_node) + root.append(stage_settings_node) + + # ------------ + # BinaryResult + # ------------ + binary_result_node = ET.Element("BinaryResult") + # ImageSize + image_size_node = ET.Element("ImageSize") + pixels_x_node = ET.Element("X") + pixels_x_node.text = str(pixels_x) + image_size_node.append(pixels_x_node) + pixels_y_node = ET.Element("Y") + pixels_y_node.text = str(pixels_y) + image_size_node.append(pixels_y_node) + binary_result_node.append(image_size_node) + # PixelSize + pixel_size_node = ET.Element("PixelSize") + pixel_size_x_node = ET.Element("X") + pixel_size_x_node.text = str(pixel_size_x) + pixel_size_node.append(pixel_size_x_node) + pixel_size_y_node = ET.Element("Y") + pixel_size_y_node.text = str(pixel_size_y) + pixel_size_node.append(pixel_size_y_node) + binary_result_node.append(pixel_size_node) + + root.append(binary_result_node) + + xml_string = ET.tostring(root, encoding="unicode", xml_declaration=True) + return xml_string + + +@pytest.mark.parametrize( + "test_params", + ( + ( + "Metadata", # Tag key + "2026-04-15-21-50-14_drift_corrected_image_Finer Milling - Electron Image.png", + "some_project", + 2000, # Voltage + 0, # Beam shift X + 0, # Y + 0.003072, # Field of view X + 0.002048, # Y + 0.003, # Stage X + 0.0003, # Y + 0.01, # Z + -1.309, # Rotation + -75, # Rotation offset + 0.8, # Alpha tilt + 0, # Beta tilt + 2, # Expected slot number + 3072, # Image size X + 2048, # Y + 1e-6, # Pixel size X + 1e-6, # Y + ), + ( + "Metadata", # Tag key + "2026-04-16-02-39-40_drift_corrected_image_Polishing 2 - Electron Image.png", + "another_project", + 2000, # Voltage + 0, # Beam shift X + 0, # Y + 0.003072, # Field of view X + 0.002048, # Y + -0.003, # Stage X + 0.0003, # Y + 0.01, # Z + 1.833, # Rotation + -75, # Rotation offset + 0, # Alpha tilt + 0, # Beta tilt + 2, # Expected slot number + 3072, # Image size X + 2048, # Y + 1e-6, # Pixel size X + 1e-6, # Y + ), + ), +) +def test_parse_metadata( + mocker: MockerFixture, + test_params: tuple[ + str, + str, + str, + float, + float, + float, + float, + float, + float, + float, + float, + float, + float, + float, + float, + int, + int, + int, + float, + float, + ], + visit_dir: Path, +): + # Unpack test params + ( + tag_key, + image_name, + project_name, + voltage, + shift_x, + shift_y, + len_x, + len_y, + pos_x, + pos_y, + pos_z, + rotation, + rotation_offset, + tilt_alpha, + tilt_beta, + expected_slot_number, + pixels_x, + pixels_y, + pixel_size_x, + pixel_size_y, + ) = test_params + file = ( + visit_dir + / "autotem" + / project_name + / "Sites" + / "Lamella" + / "LamellaEvaluationImages" + / image_name + ) + + # Mock the results of opening an image file + xml_string = create_lamella_evaluation_image_metadata( + voltage, + shift_x, + shift_y, + len_x, + len_y, + pos_x, + pos_y, + pos_z, + rotation, + tilt_alpha, + tilt_beta, + pixels_x, + pixels_y, + pixel_size_x, + pixel_size_y, + ) + tags = dict.fromkeys(["Metadata", "MetadataAsINI"], 0) + tags[tag_key] = xml_string + mock_image = MagicMock(text=tags) + mocker.patch( + "murfey.workflows.fib.register_lamella_evaluation_image.PIL.Image.open", + return_value=mock_image, + ) + + # Run the function and check that output is correct + parsed = _parse_metadata(file, visit_name, rotation_offset) + + assert parsed.visit_name == visit_name + assert parsed.file == file + assert parsed.voltage == voltage + assert parsed.shift_x == shift_x + assert parsed.shift_y == shift_y + assert parsed.len_x == len_x + assert parsed.len_y == len_y + assert parsed.pos_x == pos_x + assert parsed.pos_y == pos_y + assert parsed.pos_z == pos_z + assert parsed.rotation == rotation + assert parsed.tilt_alpha == tilt_alpha + assert parsed.tilt_beta == tilt_beta + assert parsed.pixels_x == pixels_x + assert parsed.pixels_y == pixels_y + assert parsed.pixel_size_x == pixel_size_x + assert parsed.pixel_size_y == pixel_size_y + assert parsed.slot_number == expected_slot_number + assert parsed.site_name == f"{project_name}--slot_{expected_slot_number}" + assert parsed.pixel_size == 0.5 * (pixel_size_x + pixel_size_y) def test_run( mocker: MockerFixture, + visit_dir: Path, ): + # Set up parameters + project_name = "some_project" + + # Mock the logger mock_logger = mocker.patch( "murfey.workflows.fib.register_lamella_evaluation_image.logger" ) + + # Mock the database call + mock_session = MagicMock(visit_name=visit_name, instrument_name=instrument_name) mock_murfey_db = MagicMock() - message = {"dummy": "dummy"} - run(message, mock_murfey_db) - mock_logger.debug.assert_called_once() + mock_murfey_db.exec.return_value.one.return_value = mock_session + + # Mock the machine config + machine_config = MachineConfig( + calibrations={ + "rotation_offset": -75.0, + } + ) + mocker.patch( + "murfey.workflows.fib.register_lamella_evaluation_image.get_machine_config", + return_value={instrument_name: machine_config}, + ) + + # Create the test image file to use + file = ( + visit_dir + / "autotem" + / project_name + / "Sites" + / "Lamella" + / "LamellaEvaluationImages" + / "2026-04-16-02-39-40_drift_corrected_image_Polishing 2 - Electron Image.png" + ) + + # Mock the results of '_parse_metadata' + metadata = FIBImageMetadata( + visit_name=visit_name, + file=file, + voltage=2000, + shift_x=0, + shift_y=0, + len_x=0.003072, + len_y=0.002048, + pos_x=-0.003, + pos_y=0.003, + pos_z=0.01, + rotation=1.833, + slot_number=2, + tilt_alpha=0, + tilt_beta=0, + pixels_x=3072, + pixels_y=2048, + pixel_size_x=1e-6, + pixel_size_y=1e-6, + ) + mocker.patch( + "murfey.workflows.fib.register_lamella_evaluation_image._parse_metadata", + return_value=metadata, + ) + + # Construct the message to pass to the function + message = { + "register": "fib.register_lamella_evaluation_image", + "session_id": session_id, + "lamella_image_file": str(file), + } + + # Run function and check that expected calls were made + result = run(message, mock_murfey_db) + mock_logger.debug.assert_called_with( + "Extracted the following metadata from the image:\n", + metadata.model_dump_json(indent=2), + ) + assert result["success"] From f5fad07894f81e7563a43ae43637d4f4beca4216 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 20 Aug 2026 09:55:04 +0100 Subject: [PATCH 4/5] Adjusted log levels --- .../workflows/fib/register_lamella_evaluation_image.py | 8 ++++++-- .../fib/test_register_lamella_evaluation_image.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index d1dd0d466..b0468bbd2 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -1,3 +1,4 @@ +import json import logging import math import xml.etree.ElementTree as ET @@ -153,6 +154,9 @@ def run( murfey_db: Session, ): # Outer try-finally block to ensure the database connection is closed + logger.info( + f"Received the following message:\n{json.dumps(message, indent=2, default=str)}" + ) try: try: # Validate incoming message @@ -189,13 +193,13 @@ def run( visit_name=visit_name, rotation_offset=rotation_offset, ) - logger.debug( + logger.info( "Extracted the following metadata from the image:\n", metadata.model_dump_json(indent=2), ) except Exception: logger.error( - f"Error extracting metadata from file {fib_info.atlas_file}", + f"Error extracting metadata from file {fib_info.lamella_image_file}", exc_info=True, ) return {"success": False, "requeue": False} diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index e42da6835..b12514e79 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -367,7 +367,7 @@ def test_run( # Run function and check that expected calls were made result = run(message, mock_murfey_db) - mock_logger.debug.assert_called_with( + mock_logger.info.assert_called_with( "Extracted the following metadata from the image:\n", metadata.model_dump_json(indent=2), ) From d0be51f8546a109135201a44a1bf7ee27b26a7e1 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 20 Aug 2026 11:06:47 +0100 Subject: [PATCH 5/5] Updated log formatting for outputs --- .../workflows/fib/register_lamella_evaluation_image.py | 4 ++-- .../workflows/fib/test_register_lamella_evaluation_image.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index b0468bbd2..a0885a6e0 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -194,8 +194,8 @@ def run( rotation_offset=rotation_offset, ) logger.info( - "Extracted the following metadata from the image:\n", - metadata.model_dump_json(indent=2), + "Extracted the following metadata from the image:\n" + f"{json.dumps(metadata.model_dump(), indent=2, default=str)}" ) except Exception: logger.error( diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index b12514e79..f7cfb1a90 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -1,3 +1,4 @@ +import json import xml.etree.ElementTree as ET from pathlib import Path from unittest.mock import MagicMock @@ -368,7 +369,7 @@ def test_run( # Run function and check that expected calls were made result = run(message, mock_murfey_db) mock_logger.info.assert_called_with( - "Extracted the following metadata from the image:\n", - metadata.model_dump_json(indent=2), + "Extracted the following metadata from the image:\n" + f"{json.dumps(metadata.model_dump(), indent=2, default=str)}" ) assert result["success"]