diff --git a/configs/datamodule/classic_bag_of_embeddings_datamodule.yaml b/configs/datamodule/classic_bag_of_embeddings_datamodule.yaml new file mode 100644 index 00000000..3a23d90c --- /dev/null +++ b/configs/datamodule/classic_bag_of_embeddings_datamodule.yaml @@ -0,0 +1,12 @@ +defaults: + - datasets@train: embeddings/bag_of_embeddings_sl_labeled + - datasets@val: embeddings/bag_of_embeddings_sl_labeled + - datasets@test: embeddings/bag_of_embeddings_labeled + - datasets@predict: embeddings/bag_of_embeddings_unlabeled + + - samplers@sampler: bag_of_tiles_stratified_weighted_random + +_target_: prostate_cancer.datamodule.ClassicBagOfTilesDataModule + +batch_size: ??? +num_workers: ??? diff --git a/configs/datamodule/datasets/embeddings/bag_of_embeddings_sl_labeled.yaml b/configs/datamodule/datasets/embeddings/bag_of_embeddings_sl_labeled.yaml new file mode 100644 index 00000000..ed8ade57 --- /dev/null +++ b/configs/datamodule/datasets/embeddings/bag_of_embeddings_sl_labeled.yaml @@ -0,0 +1,2 @@ +_target_: prostate_cancer.datamodule.datasets.SLLabeledBagOfEmbeddingsDataset +uris: ??? diff --git a/configs/experiment/predict/bag_of_embeddings/base.yaml b/configs/experiment/predict/bag_of_embeddings/base.yaml index fa5dc20b..39a2a460 100644 --- a/configs/experiment/predict/bag_of_embeddings/base.yaml +++ b/configs/experiment/predict/bag_of_embeddings/base.yaml @@ -2,7 +2,6 @@ defaults: - /experiment/predict/base - - /datamodule/bag_of_embeddings_datamodule@datamodule - _self_ slide_threshold: ${model_data.slide_threshold} @@ -10,5 +9,5 @@ model: sl_threshold: ${slide_threshold} datamodule: - batch_size: 2 - num_workers: 4 + batch_size: 4 + num_workers: 8 diff --git a/configs/experiment/predict/bag_of_embeddings/classic_base.yaml b/configs/experiment/predict/bag_of_embeddings/classic_base.yaml new file mode 100644 index 00000000..a78c6262 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/classic_base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/base + - /datamodule/classic_bag_of_embeddings_datamodule@datamodule + - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/hybrid_base.yaml b/configs/experiment/predict/bag_of_embeddings/hybrid_base.yaml new file mode 100644 index 00000000..c9e3c4e0 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/hybrid_base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/base + - /datamodule/bag_of_embeddings_datamodule@datamodule + - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/classic_base.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/classic_base.yaml new file mode 100644 index 00000000..1dbaf40d --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/pgp/classic_base.yaml @@ -0,0 +1,12 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/classic_base + - /model/pgp_classic_mil_model@model + - /model/checkpoints/mmci_tl/pgp_classic_mil@model_data + - _self_ + +datamodule: + predict: + uris: + - ${predict_data.tiles_filtered_w_pgp_uri_224} diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/slide_threshold.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/classic_slide_threshold.yaml similarity index 78% rename from configs/experiment/predict/bag_of_embeddings/pgp/slide_threshold.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/classic_slide_threshold.yaml index 11f1d550..eb302c57 100644 --- a/configs/experiment/predict/bag_of_embeddings/pgp/slide_threshold.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/classic_slide_threshold.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/slide_threshold_mil@callbacks - - /experiment/predict/bag_of_embeddings/pgp/base + - /experiment/predict/bag_of_embeddings/pgp/classic_base - /data/mmci/sl_group_1@predict_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/base.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/hybrid_base.yaml similarity index 80% rename from configs/experiment/predict/bag_of_embeddings/pgp/base.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/hybrid_base.yaml index 78f975da..f701af45 100644 --- a/configs/experiment/predict/bag_of_embeddings/pgp/base.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/hybrid_base.yaml @@ -1,7 +1,7 @@ # @package _global_ defaults: - - /experiment/predict/bag_of_embeddings/base + - /experiment/predict/bag_of_embeddings/hybrid_base - /model/pgp_hybrid_mil_model@model - /model/checkpoints/mmci_tl/pgp_hybrid_mil@model_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/slide_threshold.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/hybrid_slide_threshold.yaml similarity index 78% rename from configs/experiment/predict/bag_of_embeddings/virchow2/slide_threshold.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/hybrid_slide_threshold.yaml index dbcd62b5..dad11f6c 100644 --- a/configs/experiment/predict/bag_of_embeddings/virchow2/slide_threshold.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/hybrid_slide_threshold.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/slide_threshold_mil@callbacks - - /experiment/predict/bag_of_embeddings/virchow2/base + - /experiment/predict/bag_of_embeddings/pgp/hybrid_base - /data/mmci/sl_group_1@predict_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/base.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_base.yaml similarity index 67% rename from configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/base.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_base.yaml index 3d239b1d..1e6a8f54 100644 --- a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/base.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_base.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/mil_prediction_callback@callbacks - - /experiment/predict/bag_of_embeddings/pgp/base + - /experiment/predict/bag_of_embeddings/pgp/classic_base - _self_ use_case: "MIL predictions" diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_mmci_sl_group_1.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_mmci_sl_group_1.yaml new file mode 100644 index 00000000..a46d9ed6 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_mmci_sl_group_1.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_base + - /data/mmci/sl_group_1@predict_data + - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_mmci_tl_val.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_mmci_tl_val.yaml new file mode 100644 index 00000000..26fbb884 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_mmci_tl_val.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/pgp/mil_predictions/classic_base + - /data/mmci/tl_val@predict_data + - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/base.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_base.yaml similarity index 67% rename from configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/base.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_base.yaml index 577459d9..8eabe193 100644 --- a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/base.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_base.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/mil_prediction_callback@callbacks - - /experiment/predict/bag_of_embeddings/virchow2/base + - /experiment/predict/bag_of_embeddings/pgp/hybrid_base - _self_ use_case: "MIL predictions" diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/mmci_sl_group_1.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_mmci_sl_group_1.yaml similarity index 52% rename from configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/mmci_sl_group_1.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_mmci_sl_group_1.yaml index ad7964be..5112fa0a 100644 --- a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/mmci_sl_group_1.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_mmci_sl_group_1.yaml @@ -1,6 +1,6 @@ # @package _global_ defaults: - - /experiment/predict/bag_of_embeddings/pgp/mil_predictions/base + - /experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_base - /data/mmci/sl_group_1@predict_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/mmci_tl_val.yaml b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_mmci_tl_val.yaml similarity index 50% rename from configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/mmci_tl_val.yaml rename to configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_mmci_tl_val.yaml index c0c2aeba..2f622a9a 100644 --- a/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/mmci_tl_val.yaml +++ b/configs/experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_mmci_tl_val.yaml @@ -1,6 +1,6 @@ # @package _global_ defaults: - - /experiment/predict/bag_of_embeddings/pgp/mil_predictions/base + - /experiment/predict/bag_of_embeddings/pgp/mil_predictions/hybrid_base - /data/mmci/tl_val@predict_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/classic_base.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/classic_base.yaml new file mode 100644 index 00000000..3b22a5a4 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/classic_base.yaml @@ -0,0 +1,12 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/classic_base + - /model/virchow2_classic_mil_model@model + - /model/checkpoints/mmci_tl/virchow2_classic_mil@model_data + - _self_ + +datamodule: + predict: + uris: + - ${predict_data.tiles_filtered_w_virchow2_uri_224} diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/classic_slide_threshold.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/classic_slide_threshold.yaml new file mode 100644 index 00000000..4d58bfd2 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/classic_slide_threshold.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +defaults: + - /callbacks/slide_threshold_mil@callbacks + - /experiment/predict/bag_of_embeddings/virchow2/classic_base + - /data/mmci/sl_group_1@predict_data + - _self_ + +slide_threshold: 0.99 # My estimate for other models +use_case: "SL Threshold Estimation" diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/base.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/hybrid_base.yaml similarity index 81% rename from configs/experiment/predict/bag_of_embeddings/virchow2/base.yaml rename to configs/experiment/predict/bag_of_embeddings/virchow2/hybrid_base.yaml index cff0ca13..37be6954 100644 --- a/configs/experiment/predict/bag_of_embeddings/virchow2/base.yaml +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/hybrid_base.yaml @@ -1,7 +1,7 @@ # @package _global_ defaults: - - /experiment/predict/bag_of_embeddings/base + - /experiment/predict/bag_of_embeddings/hybrid_base - /model/virchow2_hybrid_mil_model@model - /model/checkpoints/mmci_tl/virchow2_hybrid_mil@model_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/hybrid_slide_threshold.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/hybrid_slide_threshold.yaml new file mode 100644 index 00000000..b59bc3dc --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/hybrid_slide_threshold.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +defaults: + - /callbacks/slide_threshold_mil@callbacks + - /experiment/predict/bag_of_embeddings/virchow2/hybrid_base + - /data/mmci/sl_group_1@predict_data + - _self_ + +slide_threshold: 0.99 # My estimate for other models +use_case: "SL Threshold Estimation" diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_base.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_base.yaml new file mode 100644 index 00000000..a81b0042 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_base.yaml @@ -0,0 +1,8 @@ +# @package _global_ + +defaults: + - /callbacks/mil_prediction_callback@callbacks + - /experiment/predict/bag_of_embeddings/virchow2/classic_base + - _self_ + +use_case: "MIL predictions" diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_mmci_sl_group_1.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_mmci_sl_group_1.yaml new file mode 100644 index 00000000..a59daeaa --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_mmci_sl_group_1.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_base + - /data/mmci/sl_group_1@predict_data + - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_mmci_tl_val.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_mmci_tl_val.yaml new file mode 100644 index 00000000..b7d6c054 --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_mmci_tl_val.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/bag_of_embeddings/virchow2/mil_predictions/classic_base + - /data/mmci/tl_val@predict_data + - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_base.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_base.yaml new file mode 100644 index 00000000..749c94fc --- /dev/null +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_base.yaml @@ -0,0 +1,8 @@ +# @package _global_ + +defaults: + - /callbacks/mil_prediction_callback@callbacks + - /experiment/predict/bag_of_embeddings/virchow2/hybrid_base + - _self_ + +use_case: "MIL predictions" diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/mmci_sl_group_1.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_mmci_sl_group_1.yaml similarity index 90% rename from configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/mmci_sl_group_1.yaml rename to configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_mmci_sl_group_1.yaml index fe9179fb..b6383a10 100644 --- a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/mmci_sl_group_1.yaml +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_mmci_sl_group_1.yaml @@ -1,6 +1,6 @@ # @package _global_ defaults: - - /experiment/predict/bag_of_embeddings/virchow2/mil_predictions/base + - /experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_base - /data/mmci/sl_group_1@predict_data - _self_ diff --git a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/mmci_tl_val.yaml b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_mmci_tl_val.yaml similarity index 90% rename from configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/mmci_tl_val.yaml rename to configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_mmci_tl_val.yaml index d8acc5e3..e72a3f3c 100644 --- a/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/mmci_tl_val.yaml +++ b/configs/experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_mmci_tl_val.yaml @@ -1,6 +1,6 @@ # @package _global_ defaults: - - /experiment/predict/bag_of_embeddings/virchow2/mil_predictions/base + - /experiment/predict/bag_of_embeddings/virchow2/mil_predictions/hybrid_base - /data/mmci/tl_val@predict_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/base.yaml b/configs/experiment/test/bag_of_embeddings/base.yaml index 919deb91..5d021718 100644 --- a/configs/experiment/test/bag_of_embeddings/base.yaml +++ b/configs/experiment/test/bag_of_embeddings/base.yaml @@ -2,7 +2,6 @@ defaults: - /experiment/test/base - - /datamodule/bag_of_embeddings_datamodule@datamodule - _self_ slide_threshold: ${model_data.slide_threshold} diff --git a/configs/experiment/test/bag_of_embeddings/classic_base.yaml b/configs/experiment/test/bag_of_embeddings/classic_base.yaml new file mode 100644 index 00000000..a4be3141 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/classic_base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/test/bag_of_embeddings/base + - /datamodule/classic_bag_of_embeddings_datamodule@datamodule + - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/hybrid_base.yaml b/configs/experiment/test/bag_of_embeddings/hybrid_base.yaml new file mode 100644 index 00000000..99d88861 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/hybrid_base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/test/bag_of_embeddings/base + - /datamodule/bag_of_embeddings_datamodule@datamodule + - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/pgp/classic_base.yaml b/configs/experiment/test/bag_of_embeddings/pgp/classic_base.yaml new file mode 100644 index 00000000..f7f44a44 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/pgp/classic_base.yaml @@ -0,0 +1,12 @@ +# @package _global_ + +defaults: + - /experiment/test/bag_of_embeddings/classic_base + - /model/pgp_classic_mil_model@model + - /model/checkpoints/mmci_tl/pgp_classic_mil@model_data + - _self_ + +datamodule: + test: + uris: + - ${test_data.tiles_filtered_w_pgp_uri_224} diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/tile_threshold.yaml b/configs/experiment/test/bag_of_embeddings/pgp/classic_tile_threshold.yaml similarity index 79% rename from configs/experiment/test/bag_of_embeddings/virchow2/tile_threshold.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/classic_tile_threshold.yaml index 8d5ac728..ebe77aaf 100644 --- a/configs/experiment/test/bag_of_embeddings/virchow2/tile_threshold.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/classic_tile_threshold.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/tile_threshold_mil@callbacks - - /experiment/test/bag_of_embeddings/virchow2/base + - /experiment/test/bag_of_embeddings/pgp/classic_base - /data/mmci/tl_val@test_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/pgp/base.yaml b/configs/experiment/test/bag_of_embeddings/pgp/hybrid_base.yaml similarity index 81% rename from configs/experiment/test/bag_of_embeddings/pgp/base.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/hybrid_base.yaml index 25db044e..613f5477 100644 --- a/configs/experiment/test/bag_of_embeddings/pgp/base.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/hybrid_base.yaml @@ -1,7 +1,7 @@ # @package _global_ defaults: - - /experiment/test/bag_of_embeddings/base + - /experiment/test/bag_of_embeddings/hybrid_base - /model/pgp_hybrid_mil_model@model - /model/checkpoints/mmci_tl/pgp_hybrid_mil@model_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/pgp/tile_threshold.yaml b/configs/experiment/test/bag_of_embeddings/pgp/hybrid_tile_threshold.yaml similarity index 79% rename from configs/experiment/test/bag_of_embeddings/pgp/tile_threshold.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/hybrid_tile_threshold.yaml index 1e133ddd..9ee56695 100644 --- a/configs/experiment/test/bag_of_embeddings/pgp/tile_threshold.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/hybrid_tile_threshold.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/tile_threshold_mil@callbacks - - /experiment/test/bag_of_embeddings/pgp/base + - /experiment/test/bag_of_embeddings/pgp/hybrid_base - /data/mmci/tl_val@test_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/base.yaml b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/classic_base.yaml similarity index 66% rename from configs/experiment/test/bag_of_embeddings/virchow2/test_suite/base.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/test_suite/classic_base.yaml index 2bc6d4e4..65918647 100644 --- a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/base.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/classic_base.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/combined_test_suite_mil@callbacks - - /experiment/test/bag_of_embeddings/virchow2/base + - /experiment/test/bag_of_embeddings/pgp/classic_base - _self_ use_case: "TL Test" diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/mmci_tl_test.yaml b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/classic_mmci_tl_test.yaml similarity index 52% rename from configs/experiment/test/bag_of_embeddings/virchow2/test_suite/mmci_tl_test.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/test_suite/classic_mmci_tl_test.yaml index 6bb9ae18..1fcab4ea 100644 --- a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/mmci_tl_test.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/classic_mmci_tl_test.yaml @@ -1,6 +1,6 @@ # @package _global_ defaults: - - /experiment/test/bag_of_embeddings/virchow2/test_suite/base + - /experiment/test/bag_of_embeddings/pgp/test_suite/classic_base - /data/mmci/tl_test@test_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/pgp/test_suite/base.yaml b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_base.yaml similarity index 67% rename from configs/experiment/test/bag_of_embeddings/pgp/test_suite/base.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_base.yaml index 32e8817b..e0138d48 100644 --- a/configs/experiment/test/bag_of_embeddings/pgp/test_suite/base.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_base.yaml @@ -2,7 +2,7 @@ defaults: - /callbacks/combined_test_suite_mil@callbacks - - /experiment/test/bag_of_embeddings/pgp/base + - /experiment/test/bag_of_embeddings/pgp/hybrid_base - _self_ use_case: "TL Test" diff --git a/configs/experiment/test/bag_of_embeddings/pgp/test_suite/mmci_tl_test.yaml b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_mmci_tl_test.yaml similarity index 53% rename from configs/experiment/test/bag_of_embeddings/pgp/test_suite/mmci_tl_test.yaml rename to configs/experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_mmci_tl_test.yaml index f5f43efc..a478a197 100644 --- a/configs/experiment/test/bag_of_embeddings/pgp/test_suite/mmci_tl_test.yaml +++ b/configs/experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_mmci_tl_test.yaml @@ -1,6 +1,6 @@ # @package _global_ defaults: - - /experiment/test/bag_of_embeddings/pgp/test_suite/base + - /experiment/test/bag_of_embeddings/pgp/test_suite/hybrid_base - /data/mmci/tl_test@test_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/classic_base.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/classic_base.yaml new file mode 100644 index 00000000..d53707ce --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/classic_base.yaml @@ -0,0 +1,12 @@ +# @package _global_ + +defaults: + - /experiment/test/bag_of_embeddings/classic_base + - /model/virchow2_classic_mil_model@model + - /model/checkpoints/mmci_tl/virchow2_classic_mil@model_data + - _self_ + +datamodule: + test: + uris: + - ${test_data.tiles_filtered_w_virchow2_uri_224} diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/classic_tile_threshold.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/classic_tile_threshold.yaml new file mode 100644 index 00000000..c70913f6 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/classic_tile_threshold.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +defaults: + - /callbacks/tile_threshold_mil@callbacks + - /experiment/test/bag_of_embeddings/virchow2/classic_base + - /data/mmci/tl_val@test_data + - _self_ + +tile_threshold: 0.9 # Pathologist's estimate for other models +use_case: "TL Threshold Estimation" diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/base.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/hybrid_base.yaml similarity index 82% rename from configs/experiment/test/bag_of_embeddings/virchow2/base.yaml rename to configs/experiment/test/bag_of_embeddings/virchow2/hybrid_base.yaml index c90e5378..c226c019 100644 --- a/configs/experiment/test/bag_of_embeddings/virchow2/base.yaml +++ b/configs/experiment/test/bag_of_embeddings/virchow2/hybrid_base.yaml @@ -1,7 +1,7 @@ # @package _global_ defaults: - - /experiment/test/bag_of_embeddings/base + - /experiment/test/bag_of_embeddings/hybrid_base - /model/virchow2_hybrid_mil_model@model - /model/checkpoints/mmci_tl/virchow2_hybrid_mil@model_data - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/hybrid_tile_threshold.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/hybrid_tile_threshold.yaml new file mode 100644 index 00000000..6ca0cfc1 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/hybrid_tile_threshold.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +defaults: + - /callbacks/tile_threshold_mil@callbacks + - /experiment/test/bag_of_embeddings/virchow2/hybrid_base + - /data/mmci/tl_val@test_data + - _self_ + +tile_threshold: 0.9 # Pathologist's estimate for other models +use_case: "TL Threshold Estimation" diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/classic_base.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/classic_base.yaml new file mode 100644 index 00000000..52e1e964 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/classic_base.yaml @@ -0,0 +1,8 @@ +# @package _global_ + +defaults: + - /callbacks/combined_test_suite_mil@callbacks + - /experiment/test/bag_of_embeddings/virchow2/classic_base + - _self_ + +use_case: "TL Test" diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/classic_mmci_tl_test.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/classic_mmci_tl_test.yaml new file mode 100644 index 00000000..b9cf0792 --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/classic_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/test/bag_of_embeddings/virchow2/test_suite/classic_base + - /data/mmci/tl_test@test_data + - _self_ diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_base.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_base.yaml new file mode 100644 index 00000000..44f929fc --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_base.yaml @@ -0,0 +1,8 @@ +# @package _global_ + +defaults: + - /callbacks/combined_test_suite_mil@callbacks + - /experiment/test/bag_of_embeddings/virchow2/hybrid_base + - _self_ + +use_case: "TL Test" diff --git a/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_mmci_tl_test.yaml b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_mmci_tl_test.yaml new file mode 100644 index 00000000..157bdbac --- /dev/null +++ b/configs/experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/test/bag_of_embeddings/virchow2/test_suite/hybrid_base + - /data/mmci/tl_test@test_data + - _self_ diff --git a/configs/experiment/train/bag_of_embeddings/base.yaml b/configs/experiment/train/bag_of_embeddings/base.yaml index 8c3a995f..627f0db4 100644 --- a/configs/experiment/train/bag_of_embeddings/base.yaml +++ b/configs/experiment/train/bag_of_embeddings/base.yaml @@ -4,7 +4,6 @@ defaults: - /experiment/train/base - /data/mmci/tl_train@train_data - /data/mmci/tl_val@val_data - - /datamodule/bag_of_embeddings_datamodule@datamodule - _self_ model: diff --git a/configs/experiment/train/bag_of_embeddings/classic_base.yaml b/configs/experiment/train/bag_of_embeddings/classic_base.yaml new file mode 100644 index 00000000..65cd3e8a --- /dev/null +++ b/configs/experiment/train/bag_of_embeddings/classic_base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/train/bag_of_embeddings/base + - /datamodule/classic_bag_of_embeddings_datamodule@datamodule + - _self_ diff --git a/configs/experiment/train/bag_of_embeddings/classic_pgp.yaml b/configs/experiment/train/bag_of_embeddings/classic_pgp.yaml new file mode 100644 index 00000000..9d38d6d3 --- /dev/null +++ b/configs/experiment/train/bag_of_embeddings/classic_pgp.yaml @@ -0,0 +1,16 @@ +# @package _global_ + +defaults: + - /experiment/train/bag_of_embeddings/classic_base + - /model/pgp_classic_mil_model@model + - _self_ + +datamodule: + train: + uris: + - ${train_data.tiles_filtered_w_pgp_uri_224} + val: + uris: + - ${val_data.tiles_filtered_w_pgp_uri_224} + +model_name: "PGP (Classic-MIL)" diff --git a/configs/experiment/train/bag_of_embeddings/classic_virchow2.yaml b/configs/experiment/train/bag_of_embeddings/classic_virchow2.yaml new file mode 100644 index 00000000..5c62bc7a --- /dev/null +++ b/configs/experiment/train/bag_of_embeddings/classic_virchow2.yaml @@ -0,0 +1,16 @@ +# @package _global_ + +defaults: + - /experiment/train/bag_of_embeddings/classic_base + - /model/virchow2_classic_mil_model@model + - _self_ + +datamodule: + train: + uris: + - ${train_data.tiles_filtered_w_virchow2_uri_224} + val: + uris: + - ${val_data.tiles_filtered_w_virchow2_uri_224} + +model_name: "Virchow2 (Classic-MIL)" diff --git a/configs/experiment/train/bag_of_embeddings/hybrid_base.yaml b/configs/experiment/train/bag_of_embeddings/hybrid_base.yaml new file mode 100644 index 00000000..40327814 --- /dev/null +++ b/configs/experiment/train/bag_of_embeddings/hybrid_base.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/train/bag_of_embeddings/base + - /datamodule/bag_of_embeddings_datamodule@datamodule + - _self_ diff --git a/configs/experiment/train/bag_of_embeddings/pgp.yaml b/configs/experiment/train/bag_of_embeddings/hybrid_pgp.yaml similarity index 83% rename from configs/experiment/train/bag_of_embeddings/pgp.yaml rename to configs/experiment/train/bag_of_embeddings/hybrid_pgp.yaml index 8c4439d0..1f04a336 100644 --- a/configs/experiment/train/bag_of_embeddings/pgp.yaml +++ b/configs/experiment/train/bag_of_embeddings/hybrid_pgp.yaml @@ -1,7 +1,7 @@ # @package _global_ defaults: - - /experiment/train/bag_of_embeddings/base + - /experiment/train/bag_of_embeddings/hybrid_base - /model/pgp_hybrid_mil_model@model - _self_ diff --git a/configs/experiment/train/bag_of_embeddings/virchow2.yaml b/configs/experiment/train/bag_of_embeddings/hybrid_virchow2.yaml similarity index 84% rename from configs/experiment/train/bag_of_embeddings/virchow2.yaml rename to configs/experiment/train/bag_of_embeddings/hybrid_virchow2.yaml index 15adeb39..6cb5d81a 100644 --- a/configs/experiment/train/bag_of_embeddings/virchow2.yaml +++ b/configs/experiment/train/bag_of_embeddings/hybrid_virchow2.yaml @@ -1,7 +1,7 @@ # @package _global_ defaults: - - /experiment/train/bag_of_embeddings/base + - /experiment/train/bag_of_embeddings/hybrid_base - /model/virchow2_hybrid_mil_model@model - _self_ diff --git a/configs/model/checkpoints/mmci_tl/pgp_classic_mil.yaml b/configs/model/checkpoints/mmci_tl/pgp_classic_mil.yaml new file mode 100644 index 00000000..371a74ed --- /dev/null +++ b/configs/model/checkpoints/mmci_tl/pgp_classic_mil.yaml @@ -0,0 +1,9 @@ +defaults: + - base_foundation + - _self_ + +tile_threshold: 0.65 +slide_threshold: 0.1796019971 + +checkpoint: "mlflow-artifacts:/65/9e9e6ec6817844c296004f0dc7369d1f/artifacts/checkpoints/epoch=6-step=1221/checkpoint.ckpt" +model_name: "PGP (Classic-MIL)" diff --git a/configs/model/checkpoints/mmci_tl/virchow2_classic_mil.yaml b/configs/model/checkpoints/mmci_tl/virchow2_classic_mil.yaml new file mode 100644 index 00000000..781c1a43 --- /dev/null +++ b/configs/model/checkpoints/mmci_tl/virchow2_classic_mil.yaml @@ -0,0 +1,9 @@ +defaults: + - base_foundation + - _self_ + +tile_threshold: 0.77 +slide_threshold: 0.1469607204 + +checkpoint: "mlflow-artifacts:/65/8613d45c98664f63b5cdd7e40e190986/artifacts/checkpoints/epoch=5-step=950/checkpoint.ckpt" +model_name: "Virchow2 (Classic-MIL)" diff --git a/configs/model/pgp_classic_mil_model.yaml b/configs/model/pgp_classic_mil_model.yaml new file mode 100644 index 00000000..42edbe12 --- /dev/null +++ b/configs/model/pgp_classic_mil_model.yaml @@ -0,0 +1,5 @@ +_target_: prostate_cancer.classic_mil_model.ProstateCancerClassicMIL +foundation: "pgp" +lr: ??? +tl_threshold: ??? +sl_threshold: ??? diff --git a/configs/model/virchow2_classic_mil_model.yaml b/configs/model/virchow2_classic_mil_model.yaml new file mode 100644 index 00000000..621bca68 --- /dev/null +++ b/configs/model/virchow2_classic_mil_model.yaml @@ -0,0 +1,5 @@ +_target_: prostate_cancer.classic_mil_model.ProstateCancerClassicMIL +foundation: "virchow2" +lr: ??? +tl_threshold: ??? +sl_threshold: ??? diff --git a/prostate_cancer/attention_mil_model.py b/prostate_cancer/attention_mil_model.py index dd424281..1f719289 100644 --- a/prostate_cancer/attention_mil_model.py +++ b/prostate_cancer/attention_mil_model.py @@ -3,134 +3,37 @@ from copy import deepcopy import torch -from lightning import LightningModule from torch import Tensor, nn -from torch.optim.adamw import AdamW -from torch.optim.optimizer import Optimizer -from torchmetrics import Metric, MetricCollection -from torchmetrics.classification import ( - AUROC, - Accuracy, - NegativePredictiveValue, - Precision, - Recall, - Specificity, -) - -from prostate_cancer.typing import ( - LabeledBagOfTilesSampleBatch, - MILModelOutput, - UnlabeledBagOfTilesSampleBatch, -) - - -class ProstateCancerAttentionMIL(LightningModule): +from torchmetrics import MetricCollection + +from prostate_cancer.mil_model_base import ProstateCancerMILBase, binary_metrics +from prostate_cancer.typing import LabeledBagOfTilesSampleBatch + + +class ProstateCancerAttentionMIL(ProstateCancerMILBase): + """Hybrid MIL: trained on both slide-level (SL) and tile-level (TL) labels.""" + def __init__( self, foundation: str, lr: float, tl_threshold: float, sl_threshold: float ) -> None: - super().__init__() - match foundation: - case "pgp": - self.input_dim = 1536 - case "virchow2": - self.input_dim = 2560 - case _: - raise ValueError(f"Unknown foundation model: {foundation}") - - self.input_dim_sqrt = torch.tensor(self.input_dim).sqrt() - - # if we did not precompute the embeddings, we would obtain it from this module - # (idendity replaced with foundation model) - self.encoder = nn.Identity() - - # from a paper - self.attention = nn.Sequential( - nn.Linear(self.input_dim, 512), - nn.Tanh(), - nn.Linear(512, 1), + super().__init__( + foundation=foundation, + lr=lr, + sl_threshold=sl_threshold, + tl_threshold=tl_threshold, ) - # TL Classifier - self.classifier = nn.Linear(self.input_dim, 1) - - self.sl_criterion = nn.BCEWithLogitsLoss(reduction="mean") self.tl_criterion = nn.BCEWithLogitsLoss( reduction="none", pos_weight=torch.tensor([9.65]) ) # handle padding - self.lr = lr - - metrics: dict[str, dict[str, Metric | MetricCollection]] = {} - - # both SL and TL metrics - for task_type, t in [("tl", tl_threshold), ("sl", sl_threshold)]: - metrics[task_type] = { - "AUC": AUROC("binary"), - "accuracy": Accuracy("binary", threshold=t), - "precision": Precision("binary", threshold=t), - "recall": Recall("binary", threshold=t), - "specificity": Specificity("binary", threshold=t), - "negative_predictive_value": NegativePredictiveValue( - "binary", threshold=t - ), - } - - self.train_metrics_sl = MetricCollection( - deepcopy(metrics["sl"]), prefix="sl_train/" - ) - self.val_metrics_sl = MetricCollection( - deepcopy(metrics["sl"]), prefix="sl_validation/" - ) - self.test_metrics_sl = MetricCollection( - deepcopy(metrics["sl"]), prefix="sl_test/" - ) + tl_metrics = binary_metrics(tl_threshold) self.train_metrics_tl = MetricCollection( - deepcopy(metrics["tl"]), prefix="tl_train/" + deepcopy(tl_metrics), prefix="tl_train/" ) self.val_metrics_tl = MetricCollection( - deepcopy(metrics["tl"]), prefix="tl_validation/" + deepcopy(tl_metrics), prefix="tl_validation/" ) - self.test_metrics_tl = MetricCollection( - deepcopy(metrics["tl"]), prefix="tl_test/" - ) - - def forward(self, x: Tensor) -> MILModelOutput: - # x has shape (batch_size, num_tiles_padded, embedding_dim) - - # Just identity - x = self.encoder(x) # (batch_size, num_tiles_padded, embedding_dim) - - # Do not attend to padded tiles (true for non-padded elements) - mask = ( - (x.abs() > 1e-6).any(dim=-1, keepdim=True).float() - ) # (batch_size, num_tiles_padded, 1) - - # TL weights (which tiles to attend to) - raw_attn: Tensor = self.attention(x) # (batch_size, num_tiles_padded, 1) - raw_attn = raw_attn.masked_fill( - ~mask.bool(), float("-inf") - ) # (batch_size, num_tiles_padded, 1) - - # make it a distribution - attention_weights = torch.softmax( - raw_attn, dim=1 - ) # (batch_size, num_tiles_padded, 1) - - # TL predictions - tl_preds_raw: Tensor = self.classifier(x) # (batch_size, num_tiles_padded, 1) - tl_preds_valid_raw = tl_preds_raw * mask - - # weight TL predictions with attention - sl_pred_raw = torch.sum( - attention_weights * tl_preds_valid_raw, dim=1 - ) # (batch_size, 1) - - return ( - sl_pred_raw.squeeze(-1), - tl_preds_valid_raw.squeeze(-1), - mask.squeeze(-1), - attention_weights.squeeze(-1), - ) # (batch_size,), (batch_size, num_tiles_padded), (batch_size, num_tiles_padded), (batch_size, num_tiles_padded) def training_step(self, batch: LabeledBagOfTilesSampleBatch) -> Tensor: # bag ~ all embeddings from a single slide @@ -153,8 +56,10 @@ def training_step(self, batch: LabeledBagOfTilesSampleBatch) -> Tensor: "train/tl_loss", tl_loss, on_step=True, prog_bar=True, batch_size=len(bags) ) - self.train_metrics_sl.update(sl_outputs, sl_labels) - self.train_metrics_tl.update(tl_outputs[mask.bool()], tl_labels[mask.bool()]) + self.train_metrics_sl.update(sl_outputs.sigmoid(), sl_labels) + self.train_metrics_tl.update( + tl_outputs[mask.bool()].sigmoid(), tl_labels[mask.bool()] + ) self.log_dict( self.train_metrics_sl, on_epoch=True, on_step=False, batch_size=len(bags) @@ -193,8 +98,10 @@ def validation_step(self, batch: LabeledBagOfTilesSampleBatch) -> None: batch_size=len(bags), ) - self.val_metrics_sl.update(sl_outputs, sl_labels) - self.val_metrics_tl.update(tl_outputs[mask.bool()], tl_labels[mask.bool()]) + self.val_metrics_sl.update(sl_outputs.sigmoid(), sl_labels) + self.val_metrics_tl.update( + tl_outputs[mask.bool()].sigmoid(), tl_labels[mask.bool()] + ) self.log_dict( self.val_metrics_sl, on_epoch=True, on_step=False, batch_size=len(bags) @@ -202,26 +109,3 @@ def validation_step(self, batch: LabeledBagOfTilesSampleBatch) -> None: self.log_dict( self.val_metrics_tl, on_epoch=True, on_step=False, batch_size=len(bags) ) - - def test_step(self, batch: LabeledBagOfTilesSampleBatch) -> MILModelOutput: # type: ignore[override] - bags, tl_labels, sl_labels, _ = batch - - sl_outputs, tl_outputs, mask, attention = self(bags) - - self.test_metrics_sl.update(sl_outputs, sl_labels) - self.test_metrics_tl.update(tl_outputs[mask.bool()], tl_labels[mask.bool()]) - - self.log_dict( - self.test_metrics_sl, on_epoch=True, on_step=False, batch_size=len(bags) - ) - self.log_dict( - self.test_metrics_tl, on_epoch=True, on_step=False, batch_size=len(bags) - ) - return sl_outputs.sigmoid(), tl_outputs.sigmoid(), mask, attention - - def predict_step(self, batch: UnlabeledBagOfTilesSampleBatch) -> MILModelOutput: - sl_preds_raw, tl_preds_raw, mask, attention = self(batch[0]) - return sl_preds_raw.sigmoid(), tl_preds_raw.sigmoid(), mask, attention - - def configure_optimizers(self) -> Optimizer: - return AdamW(self.parameters(), lr=self.lr) diff --git a/prostate_cancer/classic_mil_model.py b/prostate_cancer/classic_mil_model.py new file mode 100644 index 00000000..33e7ffb1 --- /dev/null +++ b/prostate_cancer/classic_mil_model.py @@ -0,0 +1,43 @@ +from torch import Tensor + +from prostate_cancer.mil_model_base import ProstateCancerMILBase +from prostate_cancer.typing import SLLabeledBagOfTilesSampleBatch + + +class ProstateCancerClassicMIL(ProstateCancerMILBase): + """Classic MIL: trained only on slide-level (SL) labels, no TL supervision. + + `test_step` (SL + TL metrics) and the architecture are inherited unchanged + from `ProstateCancerMILBase` - TL ground truth is still used to evaluate + the (unsupervised) per-tile classifier at test time, it just never + contributes to the training loss here. + """ + + def training_step(self, batch: SLLabeledBagOfTilesSampleBatch) -> Tensor: + # bag ~ all embeddings from a single slide + bags, sl_labels, _ = batch + + sl_outputs, _, _, _ = self(bags) + loss = self.sl_criterion(sl_outputs, sl_labels) + + self.log("train/loss", loss, on_step=True, prog_bar=True, batch_size=len(bags)) + + self.train_metrics_sl.update(sl_outputs.sigmoid(), sl_labels) + self.log_dict( + self.train_metrics_sl, on_epoch=True, on_step=False, batch_size=len(bags) + ) + + return loss + + def validation_step(self, batch: SLLabeledBagOfTilesSampleBatch) -> None: + bags, sl_labels, _ = batch + + sl_outputs, _, _, _ = self(bags) + loss = self.sl_criterion(sl_outputs, sl_labels) + + self.log("validation/loss", loss, prog_bar=True, batch_size=len(bags)) + + self.val_metrics_sl.update(sl_outputs.sigmoid(), sl_labels) + self.log_dict( + self.val_metrics_sl, on_epoch=True, on_step=False, batch_size=len(bags) + ) diff --git a/prostate_cancer/datamodule/__init__.py b/prostate_cancer/datamodule/__init__.py index 7cd2bd5c..d733127a 100644 --- a/prostate_cancer/datamodule/__init__.py +++ b/prostate_cancer/datamodule/__init__.py @@ -1,5 +1,8 @@ -from prostate_cancer.datamodule.bag_of_tiles_data_module import BagOfTilesDataModule +from prostate_cancer.datamodule.bag_of_tiles_data_module import ( + BagOfTilesDataModule, + ClassicBagOfTilesDataModule, +) from prostate_cancer.datamodule.tile_data_module import TileDataModule -__all__ = ["BagOfTilesDataModule", "TileDataModule"] +__all__ = ["BagOfTilesDataModule", "ClassicBagOfTilesDataModule", "TileDataModule"] diff --git a/prostate_cancer/datamodule/bag_of_tiles_data_module.py b/prostate_cancer/datamodule/bag_of_tiles_data_module.py index 3879b1ed..df3a09d4 100644 --- a/prostate_cancer/datamodule/bag_of_tiles_data_module.py +++ b/prostate_cancer/datamodule/bag_of_tiles_data_module.py @@ -1,5 +1,6 @@ -from collections.abc import Iterable -from typing import TYPE_CHECKING, cast +from abc import ABC +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any, cast import torch from hydra.utils import instantiate @@ -10,19 +11,76 @@ if TYPE_CHECKING: from prostate_cancer.datamodule.datasets import ( - LabeledBagOfEmbeddingsDataset, + BagOfEmbeddingsDataset, UnlabeledBagOfEmbeddingsDataset, ) from prostate_cancer.typing import ( LabeledBagOfTilesSample, LabeledBagOfTilesSampleBatch, + SLLabeledBagOfTilesSample, + SLLabeledBagOfTilesSampleBatch, UnlabeledBagOfTilesSample, UnlabeledBagOfTilesSampleBatch, ) -class BagOfTilesDataModule(LightningDataModule): +def collate_fn_labeled( + batch: list[LabeledBagOfTilesSample], +) -> LabeledBagOfTilesSampleBatch: + inputs = [] + sl_labels = [] + tl_labels = [] + metadatas = [] + for input, sl_label, tl_label, metadata in batch: + inputs.append(input) + sl_labels.append(sl_label) + tl_labels.append(tl_label) + metadatas.append(metadata) + + inputs_tensor = torch.stack(inputs) + sl_labels_tensor = torch.stack(sl_labels) + tl_labels_tensor = torch.stack(tl_labels) + return inputs_tensor, sl_labels_tensor, tl_labels_tensor, metadatas + + +def collate_fn_sl_labeled( + batch: list[SLLabeledBagOfTilesSample], +) -> SLLabeledBagOfTilesSampleBatch: + inputs = [] + sl_labels = [] + metadatas = [] + for input, sl_label, metadata in batch: + inputs.append(input) + sl_labels.append(sl_label) + metadatas.append(metadata) + + inputs_tensor = torch.stack(inputs) + sl_labels_tensor = torch.stack(sl_labels) + return inputs_tensor, sl_labels_tensor, metadatas + + +def collate_fn_unlabeled( + batch: list[UnlabeledBagOfTilesSample], +) -> UnlabeledBagOfTilesSampleBatch: + inputs = [] + metadatas = [] + for input, metadata in batch: + inputs.append(input) + metadatas.append(metadata) + inputs_tensor = torch.stack(inputs) + return inputs_tensor, metadatas + + +class BaseBagOfTilesDataModule(LightningDataModule, ABC): + """Shared plumbing for bag-of-tiles (MIL) datamodules.""" + + # set by subclasses to the collate_fn matching their train/val dataset's labels + _collate_train_val: Callable[[list[Any]], Any] + + # test is always evaluated against SL+TL ground truth, regardless of subclass + _collate_test = staticmethod(collate_fn_labeled) + def __init__( self, batch_size: int, @@ -40,22 +98,18 @@ def setup(self, stage: str) -> None: match stage: case "fit": self.train = cast( - "LabeledBagOfEmbeddingsDataset", - instantiate(self.datasets["train"]), + "BagOfEmbeddingsDataset[Any]", instantiate(self.datasets["train"]) ) self.val = cast( - "LabeledBagOfEmbeddingsDataset", - instantiate(self.datasets["val"]), + "BagOfEmbeddingsDataset[Any]", instantiate(self.datasets["val"]) ) case "val": self.val = cast( - "LabeledBagOfEmbeddingsDataset", - instantiate(self.datasets["val"]), + "BagOfEmbeddingsDataset[Any]", instantiate(self.datasets["val"]) ) case "test": self.test = cast( - "LabeledBagOfEmbeddingsDataset", - instantiate(self.datasets["test"]), + "BagOfEmbeddingsDataset[Any]", instantiate(self.datasets["test"]) ) case "predict": self.predict = cast( @@ -63,7 +117,9 @@ def setup(self, stage: str) -> None: instantiate(self.datasets["predict"]), ) - def train_dataloader(self) -> Iterable[LabeledBagOfTilesSampleBatch]: + def train_dataloader( + self, + ) -> Iterable[LabeledBagOfTilesSampleBatch | SLLabeledBagOfTilesSampleBatch]: if self.sampler_partial: sampler = instantiate(self.sampler_partial)( @@ -78,18 +134,20 @@ def train_dataloader(self) -> Iterable[LabeledBagOfTilesSampleBatch]: self.train, sampler=sampler, shuffle=shuffle, - collate_fn=collate_fn_labeled, + collate_fn=self._collate_train_val, batch_size=self.batch_size, num_workers=self.num_workers, persistent_workers=self.num_workers > 0, drop_last=True, ) - def val_dataloader(self) -> Iterable[LabeledBagOfTilesSampleBatch]: + def val_dataloader( + self, + ) -> Iterable[LabeledBagOfTilesSampleBatch | SLLabeledBagOfTilesSampleBatch]: return DataLoader( self.val, batch_size=self.batch_size, - collate_fn=collate_fn_labeled, + collate_fn=self._collate_train_val, num_workers=self.num_workers, persistent_workers=self.num_workers > 0, ) @@ -98,7 +156,7 @@ def test_dataloader(self) -> Iterable[LabeledBagOfTilesSampleBatch]: return DataLoader( self.test, batch_size=self.batch_size, - collate_fn=collate_fn_labeled, + collate_fn=self._collate_test, num_workers=self.num_workers, persistent_workers=self.num_workers > 0, ) @@ -113,32 +171,13 @@ def predict_dataloader(self) -> Iterable[UnlabeledBagOfTilesSampleBatch]: ) -def collate_fn_labeled( - batch: list[LabeledBagOfTilesSample], -) -> LabeledBagOfTilesSampleBatch: - inputs = [] - sl_labels = [] - tl_labels = [] - metadatas = [] - for input, sl_label, tl_label, metadata in batch: - inputs.append(input) - sl_labels.append(sl_label) - tl_labels.append(tl_label) - metadatas.append(metadata) +class BagOfTilesDataModule(BaseBagOfTilesDataModule): + """Datamodule for hybrid MIL: labeled samples carry both SL and TL labels.""" - inputs_tensor = torch.stack(inputs) - sl_labels_tensor = torch.stack(sl_labels) - tl_labels_tensor = torch.stack(tl_labels) - return inputs_tensor, sl_labels_tensor, tl_labels_tensor, metadatas + _collate_train_val = staticmethod(collate_fn_labeled) -def collate_fn_unlabeled( - batch: list[UnlabeledBagOfTilesSample], -) -> UnlabeledBagOfTilesSampleBatch: - inputs = [] - metadatas = [] - for input, metadata in batch: - inputs.append(input) - metadatas.append(metadata) - inputs_tensor = torch.stack(inputs) - return inputs_tensor, metadatas +class ClassicBagOfTilesDataModule(BaseBagOfTilesDataModule): + """Datamodule for classic MIL: train/val only carry SL labels.""" + + _collate_train_val = staticmethod(collate_fn_sl_labeled) diff --git a/prostate_cancer/datamodule/datasets/__init__.py b/prostate_cancer/datamodule/datasets/__init__.py index 32d3833b..4d63aae5 100644 --- a/prostate_cancer/datamodule/datasets/__init__.py +++ b/prostate_cancer/datamodule/datasets/__init__.py @@ -1,5 +1,7 @@ from prostate_cancer.datamodule.datasets.bag_of_embeddings_dataset import ( + BagOfEmbeddingsDataset, LabeledBagOfEmbeddingsDataset, + SLLabeledBagOfEmbeddingsDataset, UnlabeledBagOfEmbeddingsDataset, ) from prostate_cancer.datamodule.datasets.embeddings_dataset import ( @@ -13,9 +15,11 @@ __all__ = [ + "BagOfEmbeddingsDataset", "LabeledBagOfEmbeddingsDataset", "LabeledEmbeddingsDataset", "LabeledTilesDataset", + "SLLabeledBagOfEmbeddingsDataset", "UnlabeledBagOfEmbeddingsDataset", "UnlabeledEmbeddingsDataset", "UnlabeledTilesDataset", diff --git a/prostate_cancer/datamodule/datasets/bag_of_embeddings_dataset.py b/prostate_cancer/datamodule/datasets/bag_of_embeddings_dataset.py index a6bb5825..aadfb11a 100644 --- a/prostate_cancer/datamodule/datasets/bag_of_embeddings_dataset.py +++ b/prostate_cancer/datamodule/datasets/bag_of_embeddings_dataset.py @@ -1,5 +1,6 @@ """These Datasets were taken from Adam Kukučka Ulcerative Colitis project and modified.""" +from abc import ABC, abstractmethod from collections import Counter from collections.abc import Iterable from pathlib import Path @@ -7,43 +8,49 @@ import torch import torch.nn.functional as F +from datasets import Dataset as HFDataset from rationai.mlkit.data.datasets.slides_tiles_loader import SlidesTilesLoader from torch.utils.data import Dataset from prostate_cancer.typing import ( LabeledBagOfTilesSample, SlideMetadata, + SLLabeledBagOfTilesSample, + TilingSlideMetadata, UnlabeledBagOfTilesSample, ) -T = TypeVar("T", bound=LabeledBagOfTilesSample | UnlabeledBagOfTilesSample) +T = TypeVar( + "T", + bound=LabeledBagOfTilesSample + | SLLabeledBagOfTilesSample + | UnlabeledBagOfTilesSample, +) + + +class BagOfEmbeddingsDataset(Dataset[T], ABC, Generic[T]): + """Base for bag-of-embeddings (MIL) datasets: one item per slide. + Handles loading slide/tile metadata, assembling the (padded) bag of tile + embeddings and building the shared slide-level metadata. Subclasses only + decide which labels (if any) accompany the bag. + """ -class BagOfEmbeddingsDataset(Dataset[T], Generic[T]): def __init__( self, uris: Iterable[str], padding: bool = True, - carcinoma_roi_t: float | None = None, ) -> None: - self.include_labels = carcinoma_roi_t is not None - self.carcinoma_roi_t = carcinoma_roi_t - self._meta = SlidesTilesLoader(uris=uris) self.slides = self._meta.slides - tiles = self._meta.tiles - if self.include_labels: - tiles = tiles.map( - lambda r: { - "carcinoma": (r["carcinoma_roi_percentage"] > self.carcinoma_roi_t) - } - ) - - self.tiles = tiles - self._meta.tiles = tiles - # no need to re-build index after .map + # tiles are loaded from many sharded parquet files and concatenated, + # leaving a fragmented backing table; flatten_indices() rewrites it + # into one contiguous Arrow file so filter_tiles_by_slide()'s + # per-sample .select() isn't gathering across hundreds of shards + self.tiles = self._meta.tiles.flatten_indices() + self._meta.tiles = self.tiles self.padding = padding @@ -55,7 +62,9 @@ def __init__( def __len__(self) -> int: return len(self.slides) - def __getitem__(self, idx: int) -> T: + def _load_bag( + self, idx: int + ) -> tuple[TilingSlideMetadata, HFDataset, torch.Tensor, SlideMetadata]: slide_metadata = self.slides[idx] slide_name = Path(slide_metadata["path"]).stem @@ -81,40 +90,78 @@ def __getitem__(self, idx: int) -> T: ys=torch.tensor(slide_tiles["y"]), ) - if not self.include_labels: - return slide_embeddings, metadata # type: ignore[return-value] + return slide_metadata, slide_tiles, slide_embeddings, metadata - sl_label = torch.tensor(slide_metadata["carcinoma"]).float() + @abstractmethod + def __getitem__(self, idx: int) -> T: ... - tl_labels = torch.zeros(len(slide_embeddings)).float() - tl_labels[: len(slide_tiles)] = torch.tensor(slide_tiles["carcinoma"]).float() - return slide_embeddings, tl_labels, sl_label, metadata # type: ignore[return-value] +class UnlabeledBagOfEmbeddingsDataset( + BagOfEmbeddingsDataset[UnlabeledBagOfTilesSample] +): + def __init__( + self, + uris: Iterable[str], + padding: bool = True, + ) -> None: + super().__init__(uris=uris, padding=padding) + + def __getitem__(self, idx: int) -> UnlabeledBagOfTilesSample: + _, _, slide_embeddings, metadata = self._load_bag(idx) + return slide_embeddings, metadata -class LabeledBagOfEmbeddingsDataset(BagOfEmbeddingsDataset[LabeledBagOfTilesSample]): +class SLLabeledBagOfEmbeddingsDataset( + BagOfEmbeddingsDataset[SLLabeledBagOfTilesSample] +): + """Bag-of-embeddings dataset carrying only slide-level (SL) labels. + + Unlike `LabeledBagOfEmbeddingsDataset`, this does not require tile-level + (TL) carcinoma annotations, so it can be used with data that only has + slide-level ground truth (classic MIL, no TL supervision). + """ + def __init__( self, uris: Iterable[str], - carcinoma_roi_t: float, padding: bool = True, ) -> None: - super().__init__( - uris=uris, - padding=padding, - carcinoma_roi_t=carcinoma_roi_t, - ) + super().__init__(uris=uris, padding=padding) + def __getitem__(self, idx: int) -> SLLabeledBagOfTilesSample: + slide_metadata, _, slide_embeddings, metadata = self._load_bag(idx) + + sl_label = torch.tensor(slide_metadata["carcinoma"]).float() + + return slide_embeddings, sl_label, metadata + + +class LabeledBagOfEmbeddingsDataset(BagOfEmbeddingsDataset[LabeledBagOfTilesSample]): + """Bag-of-embeddings dataset carrying both SL and TL labels (hybrid MIL).""" -class UnlabeledBagOfEmbeddingsDataset( - BagOfEmbeddingsDataset[UnlabeledBagOfTilesSample] -): def __init__( self, uris: Iterable[str], + carcinoma_roi_t: float, padding: bool = True, ) -> None: - super().__init__( - uris=uris, - padding=padding, + super().__init__(uris=uris, padding=padding) + self.carcinoma_roi_t = carcinoma_roi_t + + self.tiles = self.tiles.map( + lambda r: { + "carcinoma": (r["carcinoma_roi_percentage"] > self.carcinoma_roi_t) + } ) + self._meta.tiles = self.tiles + # no need to re-build index after .map + + def __getitem__(self, idx: int) -> LabeledBagOfTilesSample: + slide_metadata, slide_tiles, slide_embeddings, metadata = self._load_bag(idx) + + sl_label = torch.tensor(slide_metadata["carcinoma"]).float() + + tl_labels = torch.zeros(len(slide_embeddings)).float() # pad with zero labels + tl_labels[: len(slide_tiles)] = torch.tensor(slide_tiles["carcinoma"]).float() + + return slide_embeddings, tl_labels, sl_label, metadata diff --git a/prostate_cancer/datamodule/samplers/bag_of_tiles_stratified_weighted_random_sampler.py b/prostate_cancer/datamodule/samplers/bag_of_tiles_stratified_weighted_random_sampler.py index 78f1eb2f..1969e438 100644 --- a/prostate_cancer/datamodule/samplers/bag_of_tiles_stratified_weighted_random_sampler.py +++ b/prostate_cancer/datamodule/samplers/bag_of_tiles_stratified_weighted_random_sampler.py @@ -1,15 +1,18 @@ from collections.abc import Sequence +from typing import Any from datasets import Dataset as HFDataset from torch.utils.data import WeightedRandomSampler -from prostate_cancer.datamodule.datasets import LabeledBagOfEmbeddingsDataset +from prostate_cancer.datamodule.datasets.bag_of_embeddings_dataset import ( + BagOfEmbeddingsDataset, +) class BagOfTilesStratifiedWeightedRandomSampler(WeightedRandomSampler): def __init__( self, - dataset: LabeledBagOfEmbeddingsDataset, + dataset: BagOfEmbeddingsDataset[Any], target_col: str, replacement: bool = True, ) -> None: diff --git a/prostate_cancer/mil_model_base.py b/prostate_cancer/mil_model_base.py new file mode 100644 index 00000000..7d718918 --- /dev/null +++ b/prostate_cancer/mil_model_base.py @@ -0,0 +1,157 @@ +"""Original was created by Adam Kukučka in Ulcerative Colitis project.""" + +from copy import deepcopy + +import torch +from lightning import LightningModule +from torch import Tensor, nn +from torch.optim.adamw import AdamW +from torch.optim.optimizer import Optimizer +from torchmetrics import Metric, MetricCollection +from torchmetrics.classification import ( + AUROC, + Accuracy, + NegativePredictiveValue, + Precision, + Recall, + Specificity, +) + +from prostate_cancer.typing import ( + LabeledBagOfTilesSampleBatch, + MILModelOutput, + UnlabeledBagOfTilesSampleBatch, +) + + +def binary_metrics(threshold: float) -> dict[str, Metric | MetricCollection]: + return { + "AUC": AUROC("binary"), + "accuracy": Accuracy("binary", threshold=threshold), + "precision": Precision("binary", threshold=threshold), + "recall": Recall("binary", threshold=threshold), + "specificity": Specificity("binary", threshold=threshold), + "negative_predictive_value": NegativePredictiveValue( + "binary", threshold=threshold + ), + } + + +class ProstateCancerMILBase(LightningModule): + """Attention-MIL architecture shared by hybrid (SL+TL) and classic (SL-only) models. + + The bag encoder/attention/classifier and the forward pass are identical + regardless of which labels supervise training. Both hybrid and classic + models are evaluated at test time against SL *and* TL ground truth (TL + metrics show how well the per-tile classifier localizes carcinoma even + when, for the classic model, it was never directly supervised on TL + labels) so `test_step` is shared here too. Subclasses only differ in + which labels drive `training_step`/`validation_step`. + """ + + def __init__( + self, foundation: str, lr: float, sl_threshold: float, tl_threshold: float + ) -> None: + super().__init__() + match foundation: + case "pgp": + self.input_dim = 1536 + case "virchow2": + self.input_dim = 2560 + case _: + raise ValueError(f"Unknown foundation model: {foundation}") + + self.input_dim_sqrt = torch.tensor(self.input_dim).sqrt() + + # if we did not precompute the embeddings, we would obtain it from this module + # (idendity replaced with foundation model) + self.encoder = nn.Identity() + + # from a paper + self.attention = nn.Sequential( + nn.Linear(self.input_dim, 512), + nn.Tanh(), + nn.Linear(512, 1), + ) + + # per-tile classifier, attention-pooled into the bag (SL) prediction + self.classifier = nn.Linear(self.input_dim, 1) + + self.sl_criterion = nn.BCEWithLogitsLoss(reduction="mean") + self.lr = lr + + sl_metrics = binary_metrics(sl_threshold) + self.train_metrics_sl = MetricCollection( + deepcopy(sl_metrics), prefix="sl_train/" + ) + self.val_metrics_sl = MetricCollection( + deepcopy(sl_metrics), prefix="sl_validation/" + ) + self.test_metrics_sl = MetricCollection(deepcopy(sl_metrics), prefix="sl_test/") + + # TL is only ever evaluated (not necessarily trained on) - see class docstring + self.test_metrics_tl = MetricCollection( + deepcopy(binary_metrics(tl_threshold)), prefix="tl_test/" + ) + + def forward(self, x: Tensor) -> MILModelOutput: + # x has shape (batch_size, num_tiles_padded, embedding_dim) + + # Just identity + x = self.encoder(x) # (batch_size, num_tiles_padded, embedding_dim) + + # Do not attend to padded tiles (true for non-padded elements) + mask = ( + (x.abs() > 1e-6).any(dim=-1, keepdim=True).float() + ) # (batch_size, num_tiles_padded, 1) + + # TL weights (which tiles to attend to) + raw_attn: Tensor = self.attention(x) # (batch_size, num_tiles_padded, 1) + raw_attn = raw_attn.masked_fill( + ~mask.bool(), float("-inf") + ) # (batch_size, num_tiles_padded, 1) + + # make it a distribution + attention_weights = torch.softmax( + raw_attn, dim=1 + ) # (batch_size, num_tiles_padded, 1) + + # TL predictions + tl_preds_raw: Tensor = self.classifier(x) # (batch_size, num_tiles_padded, 1) + tl_preds_valid_raw = tl_preds_raw * mask + + # weight TL predictions with attention + sl_pred_raw = torch.sum( + attention_weights * tl_preds_valid_raw, dim=1 + ) # (batch_size, 1) + + return ( + sl_pred_raw.squeeze(-1), + tl_preds_valid_raw.squeeze(-1), + mask.squeeze(-1), + attention_weights.squeeze(-1), + ) # (batch_size,), (batch_size, num_tiles_padded), (batch_size, num_tiles_padded), (batch_size, num_tiles_padded) + + def test_step(self, batch: LabeledBagOfTilesSampleBatch) -> MILModelOutput: # type: ignore[override] + bags, tl_labels, sl_labels, _ = batch + + sl_outputs, tl_outputs, mask, attention = self(bags) + sl_outputs = sl_outputs.sigmoid() + tl_outputs = tl_outputs.sigmoid() + self.test_metrics_sl.update(sl_outputs, sl_labels) + self.test_metrics_tl.update(tl_outputs[mask.bool()], tl_labels[mask.bool()]) + + self.log_dict( + self.test_metrics_sl, on_epoch=True, on_step=False, batch_size=len(bags) + ) + self.log_dict( + self.test_metrics_tl, on_epoch=True, on_step=False, batch_size=len(bags) + ) + return sl_outputs, tl_outputs, mask, attention + + def predict_step(self, batch: UnlabeledBagOfTilesSampleBatch) -> MILModelOutput: + sl_preds_raw, tl_preds_raw, mask, attention = self(batch[0]) + return sl_preds_raw.sigmoid(), tl_preds_raw.sigmoid(), mask, attention + + def configure_optimizers(self) -> Optimizer: + return AdamW(self.parameters(), lr=self.lr) diff --git a/prostate_cancer/typing.py b/prostate_cancer/typing.py index deadd360..327725d4 100644 --- a/prostate_cancer/typing.py +++ b/prostate_cancer/typing.py @@ -3,6 +3,7 @@ from torch import Tensor +# How does one row in slides.parquet look like class TilingSlideMetadata(TypedDict): id: str path: str @@ -19,6 +20,7 @@ class TilingSlideMetadata(TypedDict): carcinoma: bool +# how does tile metadata look like in TL dataset class TileMetadata(TypedDict): slide: str x: int @@ -31,13 +33,16 @@ class TileMetadataBatch(TypedDict): y: Tensor -LabeledTileSample: TypeAlias = tuple[Tensor, Tensor, TileMetadata] -UnlabeledTileSample: TypeAlias = tuple[Tensor, TileMetadata] +LabeledTileSample: TypeAlias = tuple[ + Tensor, Tensor, TileMetadata +] # Image | label | Metadata +UnlabeledTileSample: TypeAlias = tuple[Tensor, TileMetadata] # Image | Metadata LabeledTileSampleBatch: TypeAlias = tuple[Tensor, Tensor, TileMetadataBatch] UnlabeledTileSampleBatch: TypeAlias = tuple[Tensor, TileMetadataBatch] +# how does slide metadata in bag dataset look like class SlideMetadata(TypedDict): slide_id: str slide_name: str @@ -49,11 +54,15 @@ class SlideMetadata(TypedDict): LabeledBagOfTilesSample = tuple[ Tensor, Tensor, Tensor, SlideMetadata ] # tiles / embeddings, tl_labels, sl_labels, metadata +SLLabeledBagOfTilesSample = tuple[ + Tensor, Tensor, SlideMetadata +] # tiles / embeddings, sl_label, metadata (no TL labels) UnlabeledBagOfTilesSample = tuple[Tensor, SlideMetadata] # tiles / embeddings, metadata LabeledBagOfTilesSampleBatch: TypeAlias = tuple[ Tensor, Tensor, Tensor, list[SlideMetadata] ] +SLLabeledBagOfTilesSampleBatch: TypeAlias = tuple[Tensor, Tensor, list[SlideMetadata]] UnlabeledBagOfTilesSampleBatch: TypeAlias = tuple[Tensor, list[SlideMetadata]] MILModelOutput = tuple[