Skip to content

Feature/breast data #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ejdam87 merged 8 commits into from
Aug 4, 2026
Merged

Feature/breast data #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5ee5e03
feat: no dropout
ejdam87 Aug 3, 2026
1fede29
feat: initial exploration
ejdam87 Aug 3, 2026
ccbf8da
feat: exploration
ejdam87 Aug 3, 2026
0d3708e
feat: show tile masks
ejdam87 Aug 3, 2026
1a20829
feat: mammaprint tile masks
ejdam87 Aug 3, 2026
fc49eb3
feat: tile embeddings scripts
ejdam87 Aug 3, 2026
6502f12
feat: bigger files
ejdam87 Aug 4, 2026
358dec3
feat: breast data init
ejdam87 Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions configs/data/breast/base.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# @package _global_

carcinoma_artifacts: /mnt/projects/breast_cancer/bc

metadata:
experiment_name: Breast Cancer
22 changes: 22 additions & 0 deletions configs/data/breast/mmci/b20_24_base.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
defaults:
- base
- _self_


qc_masks_uri: "mlflow-artifacts:/61/1fa3adfff18b433782163a29bc9d5c81/artifacts/qc_masks_structured"
tissue_masks_uri: "mlflow-artifacts:/61/b80d4357ba3b44999896ceda971a1884/artifacts/tissue_masks"
epithelium_masks_uri: "mlflow-artifacts:/61/0af479b624824d9bae0bc830a5b99f9a/artifacts/epithelium_masks"


# were used for filtering
thresholds:
tissue_roi_t: 0.01

# during the training, epithelium is used as a weak carcinoma annotation, the threshold is risen so that wrong epithelium around corners is removed
train_pos_tissue_roi_t: 0.5
residual_t: 0.25
blur_t: 0.4
folding_t: 0.25

epithelium_roi_t: 0.2

9 changes: 9 additions & 0 deletions configs/data/breast/mmci/b20_24_full.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
defaults:
- b20_24_base
- _self_

tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_512"
tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_224"
metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_metadata.csv"

data_name: "MMCI B20-24"
11 changes: 11 additions & 0 deletions configs/data/breast/mmci/b20_24_train.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
defaults:
- b20_24_base
- _self_

tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_512"
tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_224"
tiles_filtered_uri_512: "mlflow-artifacts:/61/72b9a94ef0fa49609ea4d19d7b830329/artifacts/MMCI B20-24 Train_512"
tiles_filtered_uri_224: "mlflow-artifacts:/61/72b9a94ef0fa49609ea4d19d7b830329/artifacts/MMCI B20-24 Train_224"
metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_train_metadata.csv"

data_name: "MMCI B20-24 Train"
12 changes: 12 additions & 0 deletions configs/data/breast/mmci/b20_24_val.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
defaults:
- b20_24_base
- _self_

tiles_uri_512: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_512"
tiles_uri_224: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_224"
tiles_filtered_uri_512: "mlflow-artifacts:/61/2c4f49ecb47247ca862cdc1c1868d4b2/artifacts/MMCI B20-24 Val_512"
tiles_filtered_uri_224: "mlflow-artifacts:/61/2c4f49ecb47247ca862cdc1c1868d4b2/artifacts/MMCI B20-24 Val_224"

metadata_table: "mlflow-artifacts:/61/230c412fc727466b888fb4db3625d128/artifacts/b20_24_val_metadata.csv"

data_name: "MMCI B20-24 Val"
10 changes: 10 additions & 0 deletions configs/data/breast/mmci/base.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
defaults:
- stats
- /data/breast/base@_global_
- _self_

# TBD -> select a reference slide from train set
stains:
hematoxylin: [0.479, 0.823, 0.305]
eosin: [0.156, 0.949, 0.273]
residual: [-0.189, -0.242, 0.952]
28 changes: 28 additions & 0 deletions configs/data/breast/mmci/mammaprint_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
defaults:
- base
- _self_

qc_masks_uri: "mlflow-artifacts:/61/a382d706e2c24906a6dbcc070d6ae718/artifacts/qc_masks"
tissue_masks_uri: "mlflow-artifacts:/61/d5c1eceb2a32492ca605f33a7a13c2f0/artifacts/tissue_masks"
carcinoma_masks_uri: "mlflow-artifacts:/61/f8a6422e954d4e86b166334a3a1f75ea/artifacts/annotation_masks/carcinoma"
exclude_masks_uri: "mlflow-artifacts:/61/f8a6422e954d4e86b166334a3a1f75ea/artifacts/annotation_masks/exclude"

# were used for filtering
thresholds:
tissue_roi_t: 0.01

residual_t: 0.25
blur_t: 0.4
folding_t: 0.25

carcinoma_roi_t: 0.8
exclude_t: 0.5


tiles_uri_512: "mlflow-artifacts:/61/605e6e530d3e48ad8c8d784627b2b1d2/artifacts/mammaprint_test_512"
tiles_uri_224: "mlflow-artifacts:/61/605e6e530d3e48ad8c8d784627b2b1d2/artifacts/mammaprint_test_224"
tiles_filtered_uri_512: "mlflow-artifacts:/61/dae5ddebc8234a449ea6ca4c42b6e927/artifacts/Mammaprint Test_512"
tiles_filtered_uri_224: "mlflow-artifacts:/61/dae5ddebc8234a449ea6ca4c42b6e927/artifacts/Mammaprint Test_224"
metadata_table: "mlflow-artifacts:/61/605e6e530d3e48ad8c8d784627b2b1d2/artifacts/mammaprint_test_metadata.csv"

data_name: "Mammaprint Test"
2 changes: 2 additions & 0 deletions configs/data/breast/mmci/stats.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
mean: [221.9752, 188.3984, 222.95]
std: [41.3043, 56.8831, 29.308]
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/filter_tiles/b20_24_train.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /data/breast/mmci/b20_24_train@data
- _self_
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/filter_tiles/b20_24_val.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /data/breast/mmci/b20_24_val@data
- _self_
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/filter_tiles/mammaprint_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /data/breast/mmci/mammaprint_test@data
- _self_
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_train.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /experiment/preprocessing/tile_embeddings_v2/pgp/base
- /data/breast/mmci/b20_24_train@data
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/tile_embeddings_v2/pgp/b20_24_val.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /experiment/preprocessing/tile_embeddings_v2/pgp/base
- /data/breast/mmci/b20_24_val@data
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/tile_embeddings_v2/pgp/mammaprint_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /experiment/preprocessing/tile_embeddings_v2/pgp/base
- /data/breast/mmci/mammaprint_test@data
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_train.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /experiment/preprocessing/tile_embeddings_v2/virchow2/base
- /data/breast/mmci/b20_24_train@data
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/tile_embeddings_v2/virchow2/b20_24_val.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /experiment/preprocessing/tile_embeddings_v2/virchow2/base
- /data/breast/mmci/b20_24_val@data
5 changes: 5 additions & 0 deletions configs/experiment/preprocessing/tile_embeddings_v2/virchow2/mammaprint_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# @package _global_

defaults:
- /experiment/preprocessing/tile_embeddings_v2/virchow2/base
- /data/breast/mmci/mammaprint_test@data
15 changes: 15 additions & 0 deletions configs/experiment/preprocessing/tile_masks/b20_24_val_224.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# @package _global_

defaults:
- /data/breast/mmci/b20_24_val@data
- _self_

tile_uris:
- ${data.tiles_filtered_uri_224}

percentage_cols:
- "tissue_roi_percentage"
- "residual_percentage"
- "blur_percentage"
- "folding_percentage"
- "epithelium_roi_percentage"
15 changes: 15 additions & 0 deletions configs/experiment/preprocessing/tile_masks/b20_24_val_512.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# @package _global_

defaults:
- /data/breast/mmci/b20_24_val@data
- _self_

tile_uris:
- ${data.tiles_filtered_uri_512}

percentage_cols:
- "tissue_roi_percentage"
- "residual_percentage"
- "blur_percentage"
- "folding_percentage"
- "epithelium_roi_percentage"
16 changes: 16 additions & 0 deletions configs/experiment/preprocessing/tile_masks/mammaprint_test_512.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# @package _global_

defaults:
- /data/breast/mmci/mammaprint_test@data
- _self_

tile_uris:
- ${data.tiles_filtered_uri_512}

percentage_cols:
- "tissue_roi_percentage"
- "residual_percentage"
- "blur_percentage"
- "folding_percentage"
- "carcinoma_roi_percentage"
- "exclude_percentage"
13 changes: 13 additions & 0 deletions configs/exploration/breast/b20_24.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# @package _global_

defaults:
- /exploration/breast/base
- _self_

original_metadata: "mlflow-artifacts:/61/59fec5d041064ed1b0a7552b663f19bf/artifacts/slides_with_folds.csv"
original_tiling_512: "mlflow-artifacts:/61/8a2711c2f25346eea4057a6faa76b048/artifacts/BC_train_data"
original_tiling_224: "mlflow-artifacts:/61/e77bd9fd006e4618ac8f834411981b2a/artifacts/BC_train_data"

metadata:
run_name: B20-24 Adapted Data
description: B20-24 Adapted Data
6 changes: 6 additions & 0 deletions configs/exploration/breast/base.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# @package _global_

defaults:
- /default
- /data/breast/base@_global_
- _self_
13 changes: 13 additions & 0 deletions configs/exploration/breast/mammaprint_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# @package _global_

defaults:
- /exploration/breast/base
- _self_

original_metadata: "mlflow-artifacts:/61/cbccb8df978d4fd7bf896527274fcacc/artifacts/dataframe/dataframe_mammaprint.csv"
original_tiling_512: "mlflow-artifacts:/61/dada21791ac443d9abfca5c06e603669/artifacts/BC_test_data"
original_tiling_224: "mlflow-artifacts:/61/468b5864f5824e1486147e1bee48e424/artifacts/BC_test_data"

metadata:
run_name: Mammaprint Test Adapted Data
description: Mammaprint Test Adapted Data
1 change: 1 addition & 0 deletions configs/model/pgp_full_model.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ defaults:

decode_head:
in_features: 1536
dropout: 0.0

_target_: ml.foundation_model.FoundationCarcinomaModel
lr: ???
Expand Down
1 change: 1 addition & 0 deletions configs/model/pgp_model.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ _target_: ml.embedding_model.EmbeddingCarcinomaModel
decode_head:
_target_: ml.modeling.decode_head.BinaryEmbeddingClassifier
in_features: 1536
dropout: 0.0
lr: ???
tl_threshold: ???
1 change: 1 addition & 0 deletions configs/model/virchow2_full_model.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ defaults:

decode_head:
in_features: 2560
dropout: 0.0

_target_: ml.foundation_model.FoundationCarcinomaModel
lr: ???
Expand Down
1 change: 1 addition & 0 deletions configs/model/virchow2_model.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ _target_: ml.embedding_model.EmbeddingCarcinomaModel
decode_head:
_target_: ml.modeling.decode_head.BinaryEmbeddingClassifier
in_features: 2560
dropout: 0.0
lr: ???
tl_threshold: ???
2 changes: 1 addition & 1 deletion configs/preprocessing/tile_embeddings_v2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ encoder: ???
output_path: ${carcinoma_artifacts}/tiles_with_embeddings/${encoder}/${data.data_name}
concurrency: 64
block_size: 128
rows_per_file: 50000
rows_per_file: 150000

use_case: Tile Embeddings (V2)

Expand Down
Loading