From 5ee5e038ee05f76ee81f797944d2e49366735274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Mon, 3 Aug 2026 06:23:19 +0000 Subject: [PATCH 1/8] feat: no dropout --- configs/model/pgp_full_model.yaml | 1 + configs/model/pgp_model.yaml | 1 + configs/model/virchow2_full_model.yaml | 1 + configs/model/virchow2_model.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/configs/model/pgp_full_model.yaml b/configs/model/pgp_full_model.yaml index ca8f9673..8db2cef2 100644 --- a/configs/model/pgp_full_model.yaml +++ b/configs/model/pgp_full_model.yaml @@ -5,6 +5,7 @@ defaults: decode_head: in_features: 1536 + dropout: 0.0 _target_: ml.foundation_model.FoundationCarcinomaModel lr: ??? diff --git a/configs/model/pgp_model.yaml b/configs/model/pgp_model.yaml index a6305ae2..66303519 100644 --- a/configs/model/pgp_model.yaml +++ b/configs/model/pgp_model.yaml @@ -2,5 +2,6 @@ _target_: ml.embedding_model.EmbeddingCarcinomaModel decode_head: _target_: ml.modeling.decode_head.BinaryEmbeddingClassifier in_features: 1536 + dropout: 0.0 lr: ??? tl_threshold: ??? diff --git a/configs/model/virchow2_full_model.yaml b/configs/model/virchow2_full_model.yaml index 95a8f91c..3558896c 100644 --- a/configs/model/virchow2_full_model.yaml +++ b/configs/model/virchow2_full_model.yaml @@ -5,6 +5,7 @@ defaults: decode_head: in_features: 2560 + dropout: 0.0 _target_: ml.foundation_model.FoundationCarcinomaModel lr: ??? diff --git a/configs/model/virchow2_model.yaml b/configs/model/virchow2_model.yaml index 643836d1..ec52592c 100644 --- a/configs/model/virchow2_model.yaml +++ b/configs/model/virchow2_model.yaml @@ -2,5 +2,6 @@ _target_: ml.embedding_model.EmbeddingCarcinomaModel decode_head: _target_: ml.modeling.decode_head.BinaryEmbeddingClassifier in_features: 2560 + dropout: 0.0 lr: ??? tl_threshold: ??? From 1fede291b3fb209ac7918adfa86dc65e96a07115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Mon, 3 Aug 2026 08:14:22 +0000 Subject: [PATCH 2/8] feat: initial exploration --- configs/data/breast/base.yaml | 6 +++ configs/data/breast/mmci/b20_24_base.yaml | 26 ++++++++++++ configs/data/breast/mmci/b20_24_train.yaml | 7 ++++ configs/data/breast/mmci/b20_24_val.yaml | 7 ++++ configs/data/breast/mmci/base.yaml | 10 +++++ configs/data/breast/mmci/stats.yaml | 2 + configs/exploration/breast/b20_24.yaml | 11 +++++ configs/exploration/breast/base.yaml | 6 +++ exploration/b20_24.py | 49 ++++++++++++++++++++++ 9 files changed, 124 insertions(+) create mode 100644 configs/data/breast/base.yaml create mode 100644 configs/data/breast/mmci/b20_24_base.yaml create mode 100644 configs/data/breast/mmci/b20_24_train.yaml create mode 100644 configs/data/breast/mmci/b20_24_val.yaml create mode 100644 configs/data/breast/mmci/base.yaml create mode 100644 configs/data/breast/mmci/stats.yaml create mode 100644 configs/exploration/breast/b20_24.yaml create mode 100644 configs/exploration/breast/base.yaml create mode 100644 exploration/b20_24.py diff --git a/configs/data/breast/base.yaml b/configs/data/breast/base.yaml new file mode 100644 index 00000000..8921c69f --- /dev/null +++ b/configs/data/breast/base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +carcinoma_artifacts: /mnt/projects/breast_cancer/bc + +metadata: + experiment_name: Breast Cancer diff --git a/configs/data/breast/mmci/b20_24_base.yaml b/configs/data/breast/mmci/b20_24_base.yaml new file mode 100644 index 00000000..641bfb37 --- /dev/null +++ b/configs/data/breast/mmci/b20_24_base.yaml @@ -0,0 +1,26 @@ +defaults: + - base + - _self_ + + +qc_masks_uri: "mlflow-artifacts:/61/1fa3adfff18b433782163a29bc9d5c81/artifacts/qc_masks" +tissue_masks_uri: "mlflow-artifacts:/61/b80d4357ba3b44999896ceda971a1884/artifacts/tissue_masks" +epithelium_masks_uri: "mlflow-artifacts:/61/0af479b624824d9bae0bc830a5b99f9a/artifacts/epithelium_masks" + + +# were used for filtering +thresholds: + tissue_roi_t: 0.01 + + # during the training, epithelium is used as a weak carcinoma annotation, the threshold is risen so that wrong epithelium around corners is removed + train_pos_tissue_roi_t: 0.5 + residual_t: 0.25 + blur_t: 0.4 + folding_t: 0.25 + + epithelium_roi_t: 0.2 + +metadata_table: "mlflow-artifacts:/61/1e5b475e3b2043ba924683d457fdb37b/artifacts/b20_24_metadata.csv" + + +data_name: "MMCI B20-24" diff --git a/configs/data/breast/mmci/b20_24_train.yaml b/configs/data/breast/mmci/b20_24_train.yaml new file mode 100644 index 00000000..81301cd0 --- /dev/null +++ b/configs/data/breast/mmci/b20_24_train.yaml @@ -0,0 +1,7 @@ +defaults: + - b20_24_base + - _self_ + +metadata_table: "mlflow-artifacts:/61/1e5b475e3b2043ba924683d457fdb37b/artifacts/b20_24_train_metadata.csv" + +data_name: "MMCI B20-24 Train" diff --git a/configs/data/breast/mmci/b20_24_val.yaml b/configs/data/breast/mmci/b20_24_val.yaml new file mode 100644 index 00000000..758cb7c2 --- /dev/null +++ b/configs/data/breast/mmci/b20_24_val.yaml @@ -0,0 +1,7 @@ +defaults: + - b20_24_base + - _self_ + +metadata_table: "mlflow-artifacts:/61/1e5b475e3b2043ba924683d457fdb37b/artifacts/b20_24_val_metadata.csv" + +data_name: "MMCI B20-24 Val" diff --git a/configs/data/breast/mmci/base.yaml b/configs/data/breast/mmci/base.yaml new file mode 100644 index 00000000..f638ba8f --- /dev/null +++ b/configs/data/breast/mmci/base.yaml @@ -0,0 +1,10 @@ +defaults: + - stats + - /data/breast/base@_global_ + - _self_ + +# TBD -> select a reference slide from train set +stains: + hematoxylin: [0.479, 0.823, 0.305] + eosin: [0.156, 0.949, 0.273] + residual: [-0.189, -0.242, 0.952] diff --git a/configs/data/breast/mmci/stats.yaml b/configs/data/breast/mmci/stats.yaml new file mode 100644 index 00000000..48f1eaaa --- /dev/null +++ b/configs/data/breast/mmci/stats.yaml @@ -0,0 +1,2 @@ +mean: [221.9752, 188.3984, 222.95] +std: [41.3043, 56.8831, 29.308] diff --git a/configs/exploration/breast/b20_24.yaml b/configs/exploration/breast/b20_24.yaml new file mode 100644 index 00000000..9bcc2d07 --- /dev/null +++ b/configs/exploration/breast/b20_24.yaml @@ -0,0 +1,11 @@ +# @package _global_ + +defaults: + - /exploration/breast/base + - _self_ + +original_metadata: "mlflow-artifacts:/61/59fec5d041064ed1b0a7552b663f19bf/artifacts/slides_with_folds.csv" + +metadata: + run_name: B20-24 Data + description: B20-24 Data diff --git a/configs/exploration/breast/base.yaml b/configs/exploration/breast/base.yaml new file mode 100644 index 00000000..5ff30a82 --- /dev/null +++ b/configs/exploration/breast/base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /default + - /data/breast/base@_global_ + - _self_ diff --git a/exploration/b20_24.py b/exploration/b20_24.py new file mode 100644 index 00000000..f1b23174 --- /dev/null +++ b/exploration/b20_24.py @@ -0,0 +1,49 @@ +""" B20-24 Metadata Adapter. """ + +from pathlib import Path +from tempfile import TemporaryDirectory + +import hydra +import mlflow +import pandas as pd +from omegaconf import DictConfig +from rationai.mlkit import with_cli_args +from rationai.mlkit.autolog import autolog +from rationai.mlkit.lightning.loggers import MLFlowLogger + + +@hydra.main( + config_path="../configs", + config_name="exploration/breast/b20_24", + version_base=None, +) +@autolog +def main(config: DictConfig, logger: MLFlowLogger) -> None: + df = pd.read_csv(mlflow.artifacts.download_artifacts(config.original_metadata)) + df["carcinoma"] = df["carcinoma"].map(lambda x: x == "T") # binarize carcinoma + df["slide_path"] = df["path"] # rename path column (unified with prostate) + df = df.drop( + ["id", "path", "extent_x", "extent_y", "tile_extent_x", "tile_extent_y", "stride_x", "stride_y", "mpp_x", "mpp_y", "level", "slide_id", "total", "pos", "neg"], + axis=1 + ) # remove unnecessary attributes + + val_df = df[ df["split"] == 1 ] # use one fold as validation set (unified with prostate) + train_df = df[ df["split"] != 1 ] # use rest as training set + + with TemporaryDirectory() as tmp_dir: + target = Path(tmp_dir) / f"b20_24_metadata.csv" + df.to_csv(str(target), index=False) + + train_target = Path(tmp_dir) / f"b20_24_train_metadata.csv" + train_df.to_csv(str(train_target), index=False) + + val_target = Path(tmp_dir) / f"b20_24_val_metadata.csv" + val_df.to_csv(str(val_target), index=False) + + mlflow.log_artifact(str(target)) + mlflow.log_artifact(str(train_target)) + mlflow.log_artifact(str(val_target)) + + +if __name__ == "__main__": + main() From ccbf8dafccd0ea1feaefa48a649e9cae14f6134d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Mon, 3 Aug 2026 10:55:47 +0000 Subject: [PATCH 3/8] feat: exploration --- configs/data/breast/mmci/b20_24_base.yaml | 6 +- configs/data/breast/mmci/b20_24_full.yaml | 9 +++ configs/data/breast/mmci/b20_24_train.yaml | 4 +- configs/data/breast/mmci/b20_24_val.yaml | 4 +- configs/exploration/breast/b20_24.yaml | 6 +- exploration/b20_24.py | 84 +++++++++++++++++++--- 6 files changed, 95 insertions(+), 18 deletions(-) create mode 100644 configs/data/breast/mmci/b20_24_full.yaml diff --git a/configs/data/breast/mmci/b20_24_base.yaml b/configs/data/breast/mmci/b20_24_base.yaml index 641bfb37..dc029cf5 100644 --- a/configs/data/breast/mmci/b20_24_base.yaml +++ b/configs/data/breast/mmci/b20_24_base.yaml @@ -3,7 +3,7 @@ defaults: - _self_ -qc_masks_uri: "mlflow-artifacts:/61/1fa3adfff18b433782163a29bc9d5c81/artifacts/qc_masks" +qc_masks_uri: "mlflow-artifacts:/61/1fa3adfff18b433782163a29bc9d5c81/artifacts/qc_masks_structured" tissue_masks_uri: "mlflow-artifacts:/61/b80d4357ba3b44999896ceda971a1884/artifacts/tissue_masks" epithelium_masks_uri: "mlflow-artifacts:/61/0af479b624824d9bae0bc830a5b99f9a/artifacts/epithelium_masks" @@ -20,7 +20,3 @@ thresholds: epithelium_roi_t: 0.2 -metadata_table: "mlflow-artifacts:/61/1e5b475e3b2043ba924683d457fdb37b/artifacts/b20_24_metadata.csv" - - -data_name: "MMCI B20-24" diff --git a/configs/data/breast/mmci/b20_24_full.yaml b/configs/data/breast/mmci/b20_24_full.yaml new file mode 100644 index 00000000..97d524d8 --- /dev/null +++ b/configs/data/breast/mmci/b20_24_full.yaml @@ -0,0 +1,9 @@ +defaults: + - b20_24_base + - _self_ + +tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_512" +tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_224" +metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_metadata.csv" + +data_name: "MMCI B20-24" diff --git a/configs/data/breast/mmci/b20_24_train.yaml b/configs/data/breast/mmci/b20_24_train.yaml index 81301cd0..32e53fab 100644 --- a/configs/data/breast/mmci/b20_24_train.yaml +++ b/configs/data/breast/mmci/b20_24_train.yaml @@ -2,6 +2,8 @@ defaults: - b20_24_base - _self_ -metadata_table: "mlflow-artifacts:/61/1e5b475e3b2043ba924683d457fdb37b/artifacts/b20_24_train_metadata.csv" +tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_512" +tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_224" +metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_metadata.csv" data_name: "MMCI B20-24 Train" diff --git a/configs/data/breast/mmci/b20_24_val.yaml b/configs/data/breast/mmci/b20_24_val.yaml index 758cb7c2..175fd5a8 100644 --- a/configs/data/breast/mmci/b20_24_val.yaml +++ b/configs/data/breast/mmci/b20_24_val.yaml @@ -2,6 +2,8 @@ defaults: - b20_24_base - _self_ -metadata_table: "mlflow-artifacts:/61/1e5b475e3b2043ba924683d457fdb37b/artifacts/b20_24_val_metadata.csv" +tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_512" +tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_224" +metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_metadata.csv" data_name: "MMCI B20-24 Val" diff --git a/configs/exploration/breast/b20_24.yaml b/configs/exploration/breast/b20_24.yaml index 9bcc2d07..f0a0731f 100644 --- a/configs/exploration/breast/b20_24.yaml +++ b/configs/exploration/breast/b20_24.yaml @@ -5,7 +5,9 @@ defaults: - _self_ original_metadata: "mlflow-artifacts:/61/59fec5d041064ed1b0a7552b663f19bf/artifacts/slides_with_folds.csv" +original_tiling_512: "mlflow-artifacts:/61/8a2711c2f25346eea4057a6faa76b048/artifacts/BC_train_data" +original_tiling_224: "mlflow-artifacts:/61/e77bd9fd006e4618ac8f834411981b2a/artifacts/BC_train_data" metadata: - run_name: B20-24 Data - description: B20-24 Data + run_name: B20-24 Adapted Data + description: B20-24 Adapted Data diff --git a/exploration/b20_24.py b/exploration/b20_24.py index f1b23174..c0ec2d42 100644 --- a/exploration/b20_24.py +++ b/exploration/b20_24.py @@ -1,4 +1,4 @@ -""" B20-24 Metadata Adapter. """ +"""B20-24 Metadata Adapter.""" from pathlib import Path from tempfile import TemporaryDirectory @@ -7,9 +7,37 @@ import mlflow import pandas as pd from omegaconf import DictConfig -from rationai.mlkit import with_cli_args from rationai.mlkit.autolog import autolog from rationai.mlkit.lightning.loggers import MLFlowLogger +from rationai.tiling.writers import save_mlflow_dataset + + +def repair_split_and_log( + tiling_path: Path, all_slides: list[str], val_slide_paths: list[str], suffix: str +) -> None: + slides = pd.read_parquet(tiling_path / "slides.parquet") + tiles = pd.read_parquet(tiling_path / "tiles.parquet") + + # Binarize carcinoma the same way as the CSV metadata (T -> True, N -> False) + slides["carcinoma"] = slides["carcinoma"].map(lambda x: x == "T") + + # Keep only slides (and their tiles) that are actually referenced in the metadata CSV + all_slide_mask = slides["path"].isin(all_slides) + slides = slides[all_slide_mask] + tiles = tiles[tiles["slide_id"].isin(slides["id"])] + + # slides.path matches slide_path from the CSVs + val_slide_mask = slides["path"].isin(val_slide_paths) + val_slide_ids = slides.loc[val_slide_mask, "id"] + + train_slides, val_slides = slides[~val_slide_mask], slides[val_slide_mask] + + val_tile_mask = tiles["slide_id"].isin(val_slide_ids) + train_tiles, val_tiles = tiles[~val_tile_mask], tiles[val_tile_mask] + + save_mlflow_dataset(slides, tiles, f"b20_24_{suffix}") + save_mlflow_dataset(train_slides, train_tiles, f"b20_24_train_{suffix}") + save_mlflow_dataset(val_slides, val_tiles, f"b20_24_val_{suffix}") @hydra.main( @@ -20,15 +48,53 @@ @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: df = pd.read_csv(mlflow.artifacts.download_artifacts(config.original_metadata)) - df["carcinoma"] = df["carcinoma"].map(lambda x: x == "T") # binarize carcinoma - df["slide_path"] = df["path"] # rename path column (unified with prostate) + original_tiling_512 = Path( + mlflow.artifacts.download_artifacts(config.original_tiling_512) + ) + original_tiling_224 = Path( + mlflow.artifacts.download_artifacts(config.original_tiling_224) + ) + + df["carcinoma"] = df["carcinoma"].map(lambda x: x == "T") # binarize carcinoma + df["slide_path"] = df["path"] # rename path column (unified with prostate) df = df.drop( - ["id", "path", "extent_x", "extent_y", "tile_extent_x", "tile_extent_y", "stride_x", "stride_y", "mpp_x", "mpp_y", "level", "slide_id", "total", "pos", "neg"], - axis=1 - ) # remove unnecessary attributes + [ + "id", + "path", + "extent_x", + "extent_y", + "tile_extent_x", + "tile_extent_y", + "stride_x", + "stride_y", + "mpp_x", + "mpp_y", + "level", + "slide_id", + "total", + "pos", + "neg", + ], + axis=1, + ) # remove unnecessary attributes + + val_df = df[ + df["split"] == 1 + ] # use one fold as validation set (unified with prostate) + train_df = df[df["split"] != 1] # use rest as training set - val_df = df[ df["split"] == 1 ] # use one fold as validation set (unified with prostate) - train_df = df[ df["split"] != 1 ] # use rest as training set + repair_split_and_log( + original_tiling_512, + df["slide_path"].tolist(), + val_df["slide_path"].tolist(), + "512", + ) + repair_split_and_log( + original_tiling_224, + df["slide_path"].tolist(), + val_df["slide_path"].tolist(), + "224", + ) with TemporaryDirectory() as tmp_dir: target = Path(tmp_dir) / f"b20_24_metadata.csv" From 0d3708e8a23fdd01688be9ba639d8dcafaa9ecd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Mon, 3 Aug 2026 16:34:56 +0000 Subject: [PATCH 4/8] feat: show tile masks --- configs/data/breast/mmci/b20_24_train.yaml | 2 + configs/data/breast/mmci/b20_24_val.yaml | 3 + .../filter_tiles/b20_24_train.yaml | 5 + .../filter_tiles/b20_24_val.yaml | 5 + .../tile_masks/b20_24_val_224.yaml | 15 ++ .../tile_masks/b20_24_val_512.yaml | 15 ++ preprocessing/tiling_v2/filter_tiles.py | 2 +- pyproject.toml | 4 +- uv.lock | 158 +++++++++--------- 9 files changed, 127 insertions(+), 82 deletions(-) create mode 100644 configs/experiment/preprocessing/filter_tiles/b20_24_train.yaml create mode 100644 configs/experiment/preprocessing/filter_tiles/b20_24_val.yaml create mode 100644 configs/experiment/preprocessing/tile_masks/b20_24_val_224.yaml create mode 100644 configs/experiment/preprocessing/tile_masks/b20_24_val_512.yaml diff --git a/configs/data/breast/mmci/b20_24_train.yaml b/configs/data/breast/mmci/b20_24_train.yaml index 32e53fab..6b7d7e37 100644 --- a/configs/data/breast/mmci/b20_24_train.yaml +++ b/configs/data/breast/mmci/b20_24_train.yaml @@ -4,6 +4,8 @@ defaults: tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_512" tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_224" +tiles_filtered_uri_512: "mlflow-artifacts:/61/72b9a94ef0fa49609ea4d19d7b830329/artifacts/MMCI B20-24 Train_512" +tiles_filtered_uri_224: "mlflow-artifacts:/61/72b9a94ef0fa49609ea4d19d7b830329/artifacts/MMCI B20-24 Train_224" metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_metadata.csv" data_name: "MMCI B20-24 Train" diff --git a/configs/data/breast/mmci/b20_24_val.yaml b/configs/data/breast/mmci/b20_24_val.yaml index 175fd5a8..06725350 100644 --- a/configs/data/breast/mmci/b20_24_val.yaml +++ b/configs/data/breast/mmci/b20_24_val.yaml @@ -4,6 +4,9 @@ defaults: tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_512" tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_224" +tiles_filtered_uri_512: "mlflow-artifacts:/61/2c4f49ecb47247ca862cdc1c1868d4b2/artifacts/MMCI B20-24 Val_512" +tiles_filtered_uri_224: "mlflow-artifacts:/61/2c4f49ecb47247ca862cdc1c1868d4b2/artifacts/MMCI B20-24 Val_224" + metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_metadata.csv" data_name: "MMCI B20-24 Val" diff --git a/configs/experiment/preprocessing/filter_tiles/b20_24_train.yaml b/configs/experiment/preprocessing/filter_tiles/b20_24_train.yaml new file mode 100644 index 00000000..d4227674 --- /dev/null +++ b/configs/experiment/preprocessing/filter_tiles/b20_24_train.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /data/breast/mmci/b20_24_train@data + - _self_ diff --git a/configs/experiment/preprocessing/filter_tiles/b20_24_val.yaml b/configs/experiment/preprocessing/filter_tiles/b20_24_val.yaml new file mode 100644 index 00000000..fc6aacbb --- /dev/null +++ b/configs/experiment/preprocessing/filter_tiles/b20_24_val.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /data/breast/mmci/b20_24_val@data + - _self_ diff --git a/configs/experiment/preprocessing/tile_masks/b20_24_val_224.yaml b/configs/experiment/preprocessing/tile_masks/b20_24_val_224.yaml new file mode 100644 index 00000000..04f1c172 --- /dev/null +++ b/configs/experiment/preprocessing/tile_masks/b20_24_val_224.yaml @@ -0,0 +1,15 @@ +# @package _global_ + +defaults: + - /data/breast/mmci/b20_24_val@data + - _self_ + +tile_uris: + - ${data.tiles_filtered_uri_224} + +percentage_cols: + - "tissue_roi_percentage" + - "residual_percentage" + - "blur_percentage" + - "folding_percentage" + - "epithelium_roi_percentage" diff --git a/configs/experiment/preprocessing/tile_masks/b20_24_val_512.yaml b/configs/experiment/preprocessing/tile_masks/b20_24_val_512.yaml new file mode 100644 index 00000000..ff35486e --- /dev/null +++ b/configs/experiment/preprocessing/tile_masks/b20_24_val_512.yaml @@ -0,0 +1,15 @@ +# @package _global_ + +defaults: + - /data/breast/mmci/b20_24_val@data + - _self_ + +tile_uris: + - ${data.tiles_filtered_uri_512} + +percentage_cols: + - "tissue_roi_percentage" + - "residual_percentage" + - "blur_percentage" + - "folding_percentage" + - "epithelium_roi_percentage" diff --git a/preprocessing/tiling_v2/filter_tiles.py b/preprocessing/tiling_v2/filter_tiles.py index 1888444a..7cb54040 100644 --- a/preprocessing/tiling_v2/filter_tiles.py +++ b/preprocessing/tiling_v2/filter_tiles.py @@ -13,7 +13,7 @@ def filter_tiles(tiles: pd.DataFrame, thresholds: dict[str, int]) -> pd.DataFrame: for col in tiles.columns: - if col.endswith("percentage") and "carcinoma" not in col: + if col.endswith("percentage") and ("carcinoma" not in col) and ("epithelium" not in col): t = col.replace("percentage", "t") assert t in thresholds, f"{t} for {col}" mask = ( diff --git a/pyproject.toml b/pyproject.toml index d70316a5..c10b1b55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "prostate-cancer" +name = "carcinoma-binary-classification-methods" version = "0.1.0" -description = "A project for prostate cancer detection in digital pathology slides." +description = "A project for carcinoma detection in digital pathology slides of various tissues." readme = "README.md" requires-python = ">=3.12,<3.14" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index 2836046d..b416e2d4 100644 --- a/uv.lock +++ b/uv.lock @@ -216,6 +216,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, ] +[[package]] +name = "carcinoma-binary-classification-methods" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "albumentations" }, + { name = "datasets" }, + { name = "deprecated" }, + { name = "huggingface-hub" }, + { name = "ipykernel" }, + { name = "lightning" }, + { name = "matplotlib" }, + { name = "mlflow" }, + { name = "numpy" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "rationai-masks" }, + { name = "rationai-mlkit" }, + { name = "rationai-sdk" }, + { name = "rationai-staining" }, + { name = "rationai-tiling" }, + { name = "ratiopath" }, + { name = "ray", extra = ["data"] }, + { name = "regex" }, + { name = "scikit-learn" }, + { name = "timm" }, + { name = "torch" }, + { name = "torchvision" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "triton" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "pre-commit" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "albumentations", specifier = ">=2.0.8" }, + { name = "datasets", specifier = ">=4.6.1" }, + { name = "deprecated", specifier = ">=1.3.1" }, + { name = "huggingface-hub", specifier = ">=1.12.2" }, + { name = "ipykernel", specifier = ">=7.2.0" }, + { name = "lightning", specifier = "==2.6.0" }, + { name = "matplotlib", specifier = "==3.10.1" }, + { name = "mlflow", specifier = "==3.12.0" }, + { name = "numpy", specifier = ">=2.3.2" }, + { name = "openpyxl", specifier = "==3.1.5" }, + { name = "pandas", specifier = ">=2.2.3" }, + { name = "pillow", specifier = "==11.3.0" }, + { name = "rationai-masks", git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/masks.git" }, + { name = "rationai-mlkit", git = "https://github.com/RationAI/mlkit.git" }, + { name = "rationai-sdk", git = "https://github.com/RationAI/rationai-sdk-python.git" }, + { name = "rationai-staining", git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/staining.git?rev=2437ec9af319731f3e32aa79c4bf9e6fef5a5711" }, + { name = "rationai-tiling", git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/tiling.git" }, + { name = "ratiopath", specifier = ">=1.1.2" }, + { name = "ray", extras = ["data"], specifier = ">=2.52.1" }, + { name = "regex", specifier = ">=2026.2.28" }, + { name = "scikit-learn", specifier = "==1.8.0" }, + { name = "timm", specifier = "==1.0.15" }, + { name = "torch", specifier = ">=2.9.1" }, + { name = "torchvision", specifier = ">=0.22.0" }, + { name = "tqdm", specifier = "==4.67.1" }, + { name = "transformers", specifier = ">=5.7.0" }, + { name = "triton", specifier = ">=3.3.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy" }, + { name = "pre-commit" }, + { name = "ruff" }, +] + [[package]] name = "certifi" version = "2026.4.22" @@ -2373,85 +2452,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, ] -[[package]] -name = "prostate-cancer" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "albumentations" }, - { name = "datasets" }, - { name = "deprecated" }, - { name = "huggingface-hub" }, - { name = "ipykernel" }, - { name = "lightning" }, - { name = "matplotlib" }, - { name = "mlflow" }, - { name = "numpy" }, - { name = "openpyxl" }, - { name = "pandas" }, - { name = "pillow" }, - { name = "rationai-masks" }, - { name = "rationai-mlkit" }, - { name = "rationai-sdk" }, - { name = "rationai-staining" }, - { name = "rationai-tiling" }, - { name = "ratiopath" }, - { name = "ray", extra = ["data"] }, - { name = "regex" }, - { name = "scikit-learn" }, - { name = "timm" }, - { name = "torch" }, - { name = "torchvision" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "triton" }, -] - -[package.dev-dependencies] -dev = [ - { name = "mypy" }, - { name = "pre-commit" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "albumentations", specifier = ">=2.0.8" }, - { name = "datasets", specifier = ">=4.6.1" }, - { name = "deprecated", specifier = ">=1.3.1" }, - { name = "huggingface-hub", specifier = ">=1.12.2" }, - { name = "ipykernel", specifier = ">=7.2.0" }, - { name = "lightning", specifier = "==2.6.0" }, - { name = "matplotlib", specifier = "==3.10.1" }, - { name = "mlflow", specifier = "==3.12.0" }, - { name = "numpy", specifier = ">=2.3.2" }, - { name = "openpyxl", specifier = "==3.1.5" }, - { name = "pandas", specifier = ">=2.2.3" }, - { name = "pillow", specifier = "==11.3.0" }, - { name = "rationai-masks", git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/masks.git" }, - { name = "rationai-mlkit", git = "https://github.com/RationAI/mlkit.git" }, - { name = "rationai-sdk", git = "https://github.com/RationAI/rationai-sdk-python.git" }, - { name = "rationai-staining", git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/staining.git?rev=2437ec9af319731f3e32aa79c4bf9e6fef5a5711" }, - { name = "rationai-tiling", git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/tiling.git" }, - { name = "ratiopath", specifier = ">=1.1.2" }, - { name = "ray", extras = ["data"], specifier = ">=2.52.1" }, - { name = "regex", specifier = ">=2026.2.28" }, - { name = "scikit-learn", specifier = "==1.8.0" }, - { name = "timm", specifier = "==1.0.15" }, - { name = "torch", specifier = ">=2.9.1" }, - { name = "torchvision", specifier = ">=0.22.0" }, - { name = "tqdm", specifier = "==4.67.1" }, - { name = "transformers", specifier = ">=5.7.0" }, - { name = "triton", specifier = ">=3.3.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "mypy" }, - { name = "pre-commit" }, - { name = "ruff" }, -] - [[package]] name = "protobuf" version = "5.29.6" From 1a208295ab4f8014f78b0584fb93d34e3b9b9271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Mon, 3 Aug 2026 17:22:45 +0000 Subject: [PATCH 5/8] feat: mammaprint tile masks --- configs/data/breast/mmci/mammaprint_test.yaml | 28 ++++++++++ .../filter_tiles/mammaprint_test.yaml | 5 ++ .../tile_masks/mammaprint_test_512.yaml | 16 ++++++ .../exploration/breast/mammaprint_test.yaml | 13 +++++ exploration/b20_24.py | 2 +- exploration/mammaprint_test.py | 55 +++++++++++++++++++ 6 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 configs/data/breast/mmci/mammaprint_test.yaml create mode 100644 configs/experiment/preprocessing/filter_tiles/mammaprint_test.yaml create mode 100644 configs/experiment/preprocessing/tile_masks/mammaprint_test_512.yaml create mode 100644 configs/exploration/breast/mammaprint_test.yaml create mode 100644 exploration/mammaprint_test.py diff --git a/configs/data/breast/mmci/mammaprint_test.yaml b/configs/data/breast/mmci/mammaprint_test.yaml new file mode 100644 index 00000000..4c2844d8 --- /dev/null +++ b/configs/data/breast/mmci/mammaprint_test.yaml @@ -0,0 +1,28 @@ +defaults: + - base + - _self_ + +qc_masks_uri: "mlflow-artifacts:/61/a382d706e2c24906a6dbcc070d6ae718/artifacts/qc_masks" +tissue_masks_uri: "mlflow-artifacts:/61/d5c1eceb2a32492ca605f33a7a13c2f0/artifacts/tissue_masks" +carcinoma_masks_uri: "mlflow-artifacts:/61/f8a6422e954d4e86b166334a3a1f75ea/artifacts/annotation_masks/carcinoma" +exclude_masks_uri: "mlflow-artifacts:/61/f8a6422e954d4e86b166334a3a1f75ea/artifacts/annotation_masks/exclude" + +# were used for filtering +thresholds: + tissue_roi_t: 0.01 + + residual_t: 0.25 + blur_t: 0.4 + folding_t: 0.25 + + carcinoma_roi_t: 0.8 + exclude_t: 0.5 + + +tiles_uri_512: "mlflow-artifacts:/61/605e6e530d3e48ad8c8d784627b2b1d2/artifacts/mammaprint_test_512" +tiles_uri_224: "mlflow-artifacts:/61/605e6e530d3e48ad8c8d784627b2b1d2/artifacts/mammaprint_test_224" +tiles_filtered_uri_512: "mlflow-artifacts:/61/dae5ddebc8234a449ea6ca4c42b6e927/artifacts/Mammaprint Test_512" +tiles_filtered_uri_224: "mlflow-artifacts:/61/dae5ddebc8234a449ea6ca4c42b6e927/artifacts/Mammaprint Test_224" +metadata_table: "mlflow-artifacts:/61/605e6e530d3e48ad8c8d784627b2b1d2/artifacts/mammaprint_test_metadata.csv" + +data_name: "Mammaprint Test" diff --git a/configs/experiment/preprocessing/filter_tiles/mammaprint_test.yaml b/configs/experiment/preprocessing/filter_tiles/mammaprint_test.yaml new file mode 100644 index 00000000..da211984 --- /dev/null +++ b/configs/experiment/preprocessing/filter_tiles/mammaprint_test.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /data/breast/mmci/mammaprint_test@data + - _self_ diff --git a/configs/experiment/preprocessing/tile_masks/mammaprint_test_512.yaml b/configs/experiment/preprocessing/tile_masks/mammaprint_test_512.yaml new file mode 100644 index 00000000..f62ae761 --- /dev/null +++ b/configs/experiment/preprocessing/tile_masks/mammaprint_test_512.yaml @@ -0,0 +1,16 @@ +# @package _global_ + +defaults: + - /data/breast/mmci/mammaprint_test@data + - _self_ + +tile_uris: + - ${data.tiles_filtered_uri_512} + +percentage_cols: + - "tissue_roi_percentage" + - "residual_percentage" + - "blur_percentage" + - "folding_percentage" + - "carcinoma_roi_percentage" + - "exclude_percentage" diff --git a/configs/exploration/breast/mammaprint_test.yaml b/configs/exploration/breast/mammaprint_test.yaml new file mode 100644 index 00000000..107ae987 --- /dev/null +++ b/configs/exploration/breast/mammaprint_test.yaml @@ -0,0 +1,13 @@ +# @package _global_ + +defaults: + - /exploration/breast/base + - _self_ + +original_metadata: "mlflow-artifacts:/61/cbccb8df978d4fd7bf896527274fcacc/artifacts/dataframe/dataframe_mammaprint.csv" +original_tiling_512: "mlflow-artifacts:/61/dada21791ac443d9abfca5c06e603669/artifacts/BC_test_data" +original_tiling_224: "mlflow-artifacts:/61/468b5864f5824e1486147e1bee48e424/artifacts/BC_test_data" + +metadata: + run_name: Mammaprint Test Adapted Data + description: Mammaprint Test Adapted Data diff --git a/exploration/b20_24.py b/exploration/b20_24.py index c0ec2d42..c0cb9f94 100644 --- a/exploration/b20_24.py +++ b/exploration/b20_24.py @@ -1,4 +1,4 @@ -"""B20-24 Metadata Adapter.""" +"""B20-24 Data Adapter.""" from pathlib import Path from tempfile import TemporaryDirectory diff --git a/exploration/mammaprint_test.py b/exploration/mammaprint_test.py new file mode 100644 index 00000000..e4f7bf41 --- /dev/null +++ b/exploration/mammaprint_test.py @@ -0,0 +1,55 @@ +""" Mammaprint Test Data Adapter.""" + +from pathlib import Path +from tempfile import TemporaryDirectory + +import hydra +import mlflow +import pandas as pd +from omegaconf import DictConfig +from rationai.mlkit.autolog import autolog +from rationai.mlkit.lightning.loggers import MLFlowLogger +from rationai.tiling.writers import save_mlflow_dataset + + +def repair_and_log(tiling_path: Path, suffix: str) -> None: + slides = pd.read_parquet(tiling_path / "slides.parquet") + tiles = pd.read_parquet(tiling_path / "tiles.parquet") + slides["carcinoma"] = True # all slides are positive + save_mlflow_dataset(slides, tiles, f"mammaprint_test_{suffix}") + + +@hydra.main( + config_path="../configs", + config_name="exploration/breast/mammaprint_test", + version_base=None, +) +@autolog +def main(config: DictConfig, logger: MLFlowLogger) -> None: + df = pd.read_csv(mlflow.artifacts.download_artifacts(config.original_metadata)) + original_tiling_512 = Path( + mlflow.artifacts.download_artifacts(config.original_tiling_512) + ) + original_tiling_224 = Path( + mlflow.artifacts.download_artifacts(config.original_tiling_224) + ) + + df["carcinoma"] = True # all slides are positive + + repair_and_log( + original_tiling_512, + "512", + ) + repair_and_log( + original_tiling_224, + "224" + ) + + with TemporaryDirectory() as tmp_dir: + target = Path(tmp_dir) / f"mammaprint_test_metadata.csv" + df.to_csv(str(target), index=False) + mlflow.log_artifact(str(target)) + + +if __name__ == "__main__": + main() From fc49eb3da26c62fcd87563a506947a8673e6918e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Mon, 3 Aug 2026 17:34:26 +0000 Subject: [PATCH 6/8] feat: tile embeddings scripts --- .../preprocessing/tile_embeddings_v2/pgp/b20_24_train.yaml | 5 +++++ .../preprocessing/tile_embeddings_v2/pgp/b20_24_val.yaml | 5 +++++ .../tile_embeddings_v2/pgp/mammaprint_test.yaml | 5 +++++ .../tile_embeddings_v2/virchow2/b20_24_train.yaml | 5 +++++ .../tile_embeddings_v2/virchow2/b20_24_val.yaml | 5 +++++ .../tile_embeddings_v2/virchow2/mammaprint_test.yaml | 5 +++++ 6 files changed, 30 insertions(+) create mode 100644 configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_train.yaml create mode 100644 configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_val.yaml create mode 100644 configs/experiment/preprocessing/tile_embeddings_v2/pgp/mammaprint_test.yaml create mode 100644 configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_train.yaml create mode 100644 configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_val.yaml create mode 100644 configs/experiment/preprocessing/tile_embeddings_v2/virchow2/mammaprint_test.yaml diff --git a/configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_train.yaml b/configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_train.yaml new file mode 100644 index 00000000..704da147 --- /dev/null +++ b/configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_train.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /experiment/preprocessing/tile_embeddings_v2/pgp/base + - /data/breast/mmci/b20_24_train@data diff --git a/configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_val.yaml b/configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_val.yaml new file mode 100644 index 00000000..5d7cf4e9 --- /dev/null +++ b/configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_val.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /experiment/preprocessing/tile_embeddings_v2/pgp/base + - /data/breast/mmci/b20_24_val@data diff --git a/configs/experiment/preprocessing/tile_embeddings_v2/pgp/mammaprint_test.yaml b/configs/experiment/preprocessing/tile_embeddings_v2/pgp/mammaprint_test.yaml new file mode 100644 index 00000000..d02ebd6b --- /dev/null +++ b/configs/experiment/preprocessing/tile_embeddings_v2/pgp/mammaprint_test.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /experiment/preprocessing/tile_embeddings_v2/pgp/base + - /data/breast/mmci/mammaprint_test@data diff --git a/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_train.yaml b/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_train.yaml new file mode 100644 index 00000000..9818526d --- /dev/null +++ b/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_train.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /experiment/preprocessing/tile_embeddings_v2/virchow2/base + - /data/breast/mmci/b20_24_train@data diff --git a/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_val.yaml b/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_val.yaml new file mode 100644 index 00000000..067688bc --- /dev/null +++ b/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_val.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /experiment/preprocessing/tile_embeddings_v2/virchow2/base + - /data/breast/mmci/b20_24_val@data diff --git a/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/mammaprint_test.yaml b/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/mammaprint_test.yaml new file mode 100644 index 00000000..8b688cfb --- /dev/null +++ b/configs/experiment/preprocessing/tile_embeddings_v2/virchow2/mammaprint_test.yaml @@ -0,0 +1,5 @@ +# @package _global_ + +defaults: + - /experiment/preprocessing/tile_embeddings_v2/virchow2/base + - /data/breast/mmci/mammaprint_test@data From 6502f12511f0772f92a3aeecfb76208cc4a7abfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Tue, 4 Aug 2026 06:22:56 +0000 Subject: [PATCH 7/8] feat: bigger files --- configs/preprocessing/tile_embeddings_v2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/preprocessing/tile_embeddings_v2.yaml b/configs/preprocessing/tile_embeddings_v2.yaml index fd978d1f..561c398f 100644 --- a/configs/preprocessing/tile_embeddings_v2.yaml +++ b/configs/preprocessing/tile_embeddings_v2.yaml @@ -5,7 +5,7 @@ encoder: ??? output_path: ${carcinoma_artifacts}/tiles_with_embeddings/${encoder}/${data.data_name} concurrency: 64 block_size: 128 -rows_per_file: 50000 +rows_per_file: 150000 use_case: Tile Embeddings (V2) From 358dec3ad48864d0c71863596c108e46216441d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20D=C5=BEado=C5=88?= <524839@mail.muni.cz> Date: Tue, 4 Aug 2026 18:21:19 +0000 Subject: [PATCH 8/8] feat: breast data init --- exploration/b20_24.py | 6 +++--- exploration/mammaprint_test.py | 13 +++++-------- preprocessing/tiling_v2/filter_tiles.py | 6 +++++- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/exploration/b20_24.py b/exploration/b20_24.py index c0cb9f94..b99c598c 100644 --- a/exploration/b20_24.py +++ b/exploration/b20_24.py @@ -97,13 +97,13 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: ) with TemporaryDirectory() as tmp_dir: - target = Path(tmp_dir) / f"b20_24_metadata.csv" + target = Path(tmp_dir) / "b20_24_metadata.csv" df.to_csv(str(target), index=False) - train_target = Path(tmp_dir) / f"b20_24_train_metadata.csv" + train_target = Path(tmp_dir) / "b20_24_train_metadata.csv" train_df.to_csv(str(train_target), index=False) - val_target = Path(tmp_dir) / f"b20_24_val_metadata.csv" + val_target = Path(tmp_dir) / "b20_24_val_metadata.csv" val_df.to_csv(str(val_target), index=False) mlflow.log_artifact(str(target)) diff --git a/exploration/mammaprint_test.py b/exploration/mammaprint_test.py index e4f7bf41..1cba00c9 100644 --- a/exploration/mammaprint_test.py +++ b/exploration/mammaprint_test.py @@ -1,4 +1,4 @@ -""" Mammaprint Test Data Adapter.""" +"""Mammaprint Test Data Adapter.""" from pathlib import Path from tempfile import TemporaryDirectory @@ -15,7 +15,7 @@ def repair_and_log(tiling_path: Path, suffix: str) -> None: slides = pd.read_parquet(tiling_path / "slides.parquet") tiles = pd.read_parquet(tiling_path / "tiles.parquet") - slides["carcinoma"] = True # all slides are positive + slides["carcinoma"] = True # all slides are positive save_mlflow_dataset(slides, tiles, f"mammaprint_test_{suffix}") @@ -34,19 +34,16 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: mlflow.artifacts.download_artifacts(config.original_tiling_224) ) - df["carcinoma"] = True # all slides are positive + df["carcinoma"] = True # all slides are positive repair_and_log( original_tiling_512, "512", ) - repair_and_log( - original_tiling_224, - "224" - ) + repair_and_log(original_tiling_224, "224") with TemporaryDirectory() as tmp_dir: - target = Path(tmp_dir) / f"mammaprint_test_metadata.csv" + target = Path(tmp_dir) / "mammaprint_test_metadata.csv" df.to_csv(str(target), index=False) mlflow.log_artifact(str(target)) diff --git a/preprocessing/tiling_v2/filter_tiles.py b/preprocessing/tiling_v2/filter_tiles.py index 7cb54040..dcbbd518 100644 --- a/preprocessing/tiling_v2/filter_tiles.py +++ b/preprocessing/tiling_v2/filter_tiles.py @@ -13,7 +13,11 @@ def filter_tiles(tiles: pd.DataFrame, thresholds: dict[str, int]) -> pd.DataFrame: for col in tiles.columns: - if col.endswith("percentage") and ("carcinoma" not in col) and ("epithelium" not in col): + if ( + col.endswith("percentage") + and ("carcinoma" not in col) + and ("epithelium" not in col) + ): t = col.replace("percentage", "t") assert t in thresholds, f"{t} for {col}" mask = (