diff --git a/configs/callbacks/multi_aggregator_eval.yaml b/configs/callbacks/multi_aggregator_eval.yaml new file mode 100644 index 00000000..3ea0cf37 --- /dev/null +++ b/configs/callbacks/multi_aggregator_eval.yaml @@ -0,0 +1,8 @@ +multi_aggregator_eval_callback: + _target_: prostate_cancer.callbacks.MultiAggregatorEvalCallback + max_aggregator: ${max_aggregator} + mean_pool_max_aggregator: ${mean_pool_max_aggregator} + top_k_aggregator: ${top_k_aggregator} + max_threshold: ${model_data.slide_max_threshold} + mean_pool_max_threshold: ${model_data.slide_mpm_threshold} + top_k_threshold: ${model_data.slide_top_k_threshold} diff --git a/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/base.yaml b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/base.yaml new file mode 100644 index 00000000..20d624e1 --- /dev/null +++ b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/embeddings/pgp/base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/mmci_tl_test.yaml b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/mmci_tl_test.yaml new file mode 100644 index 00000000..6b3172c5 --- /dev/null +++ b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/embeddings/pgp/multi_aggregator_eval/base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_base.yaml b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_base.yaml new file mode 100644 index 00000000..82a24fa6 --- /dev/null +++ b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/embeddings/pgp/stain_base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 3 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 2 + extent_tile: 224 + stride_tile: 112 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_mmci_tl_test.yaml b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_mmci_tl_test.yaml new file mode 100644 index 00000000..2c9cc2e2 --- /dev/null +++ b/configs/experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/embeddings/pgp/multi_aggregator_eval/stain_base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/base.yaml b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/base.yaml new file mode 100644 index 00000000..fa2e2f87 --- /dev/null +++ b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/embeddings/virchow2/base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 2 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 2 + extent_tile: 224 + stride_tile: 112 + k: 3 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/mmci_tl_test.yaml b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/mmci_tl_test.yaml new file mode 100644 index 00000000..adf49986 --- /dev/null +++ b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/embeddings/virchow2/multi_aggregator_eval/base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_base.yaml b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_base.yaml new file mode 100644 index 00000000..cccc68b0 --- /dev/null +++ b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/embeddings/virchow2/stain_base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 2 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 2 + extent_tile: 224 + stride_tile: 112 + k: 2 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_mmci_tl_test.yaml b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_mmci_tl_test.yaml new file mode 100644 index 00000000..4856694a --- /dev/null +++ b/configs/experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/embeddings/virchow2/multi_aggregator_eval/stain_base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/google_vit/mpm_estimation.yaml b/configs/experiment/predict/images/google_vit/estimations/mpm_estimation.yaml similarity index 100% rename from configs/experiment/predict/images/google_vit/mpm_estimation.yaml rename to configs/experiment/predict/images/google_vit/estimations/mpm_estimation.yaml diff --git a/configs/experiment/predict/images/google_vit/span_estimation.yaml b/configs/experiment/predict/images/google_vit/estimations/span_estimation.yaml similarity index 100% rename from configs/experiment/predict/images/google_vit/span_estimation.yaml rename to configs/experiment/predict/images/google_vit/estimations/span_estimation.yaml diff --git a/configs/experiment/predict/images/google_vit/stain_mpm_estimation.yaml b/configs/experiment/predict/images/google_vit/estimations/stain_mpm_estimation.yaml similarity index 100% rename from configs/experiment/predict/images/google_vit/stain_mpm_estimation.yaml rename to configs/experiment/predict/images/google_vit/estimations/stain_mpm_estimation.yaml diff --git a/configs/experiment/predict/images/google_vit/stain_span_estimation.yaml b/configs/experiment/predict/images/google_vit/estimations/stain_span_estimation.yaml similarity index 100% rename from configs/experiment/predict/images/google_vit/stain_span_estimation.yaml rename to configs/experiment/predict/images/google_vit/estimations/stain_span_estimation.yaml diff --git a/configs/experiment/predict/images/google_vit/stain_top_k_estimation.yaml b/configs/experiment/predict/images/google_vit/estimations/stain_top_k_estimation.yaml similarity index 100% rename from configs/experiment/predict/images/google_vit/stain_top_k_estimation.yaml rename to configs/experiment/predict/images/google_vit/estimations/stain_top_k_estimation.yaml diff --git a/configs/experiment/predict/images/google_vit/top_k_estimation.yaml b/configs/experiment/predict/images/google_vit/estimations/top_k_estimation.yaml similarity index 100% rename from configs/experiment/predict/images/google_vit/top_k_estimation.yaml rename to configs/experiment/predict/images/google_vit/estimations/top_k_estimation.yaml diff --git a/configs/experiment/predict/images/google_vit/multi_aggregator_eval/base.yaml b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/base.yaml new file mode 100644 index 00000000..22d71d31 --- /dev/null +++ b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/google_vit/base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 + k: 2 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/google_vit/multi_aggregator_eval/mmci_tl_test.yaml b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/mmci_tl_test.yaml new file mode 100644 index 00000000..7b9e66d0 --- /dev/null +++ b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/google_vit/multi_aggregator_eval/base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/google_vit/multi_aggregator_eval/stain_base.yaml b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/stain_base.yaml new file mode 100644 index 00000000..b2efbf1c --- /dev/null +++ b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/stain_base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/google_vit/stain_base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 6 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 2 + extent_tile: 224 + stride_tile: 112 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/google_vit/multi_aggregator_eval/stain_mmci_tl_test.yaml b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/stain_mmci_tl_test.yaml new file mode 100644 index 00000000..8b0c1cc2 --- /dev/null +++ b/configs/experiment/predict/images/google_vit/multi_aggregator_eval/stain_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/google_vit/multi_aggregator_eval/stain_base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/base.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/base.yaml new file mode 100644 index 00000000..7a068823 --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/resnet50/base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 5 + extent_tile: 512 + stride_tile: 256 +top_k_aggregator: + kernel_size: 4 + extent_tile: 512 + stride_tile: 256 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/base_224.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/base_224.yaml new file mode 100644 index 00000000..3080b5f9 --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/base_224.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/resnet50/base_224 + - _self_ + +mean_pool_max_aggregator: + kernel_size: 6 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 5 + extent_tile: 224 + stride_tile: 112 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/mmci_tl_test.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/mmci_tl_test.yaml new file mode 100644 index 00000000..10b5f0dc --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/resnet50/multi_aggregator_eval/base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/mmci_tl_test_224.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/mmci_tl_test_224.yaml new file mode 100644 index 00000000..764577b3 --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/mmci_tl_test_224.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/resnet50/multi_aggregator_eval/base_224 + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_base.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_base.yaml new file mode 100644 index 00000000..1ff1771b --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/resnet50/stain_base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 3 + extent_tile: 512 + stride_tile: 256 +top_k_aggregator: + kernel_size: 1 + extent_tile: 512 + stride_tile: 256 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_base_224.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_base_224.yaml new file mode 100644 index 00000000..2b287e9c --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_base_224.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/resnet50/stain_base_224 + - _self_ + +mean_pool_max_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 + k: 2 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_mmci_tl_test.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_mmci_tl_test.yaml new file mode 100644 index 00000000..8f78a41e --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/resnet50/multi_aggregator_eval/stain_base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_mmci_tl_test_224.yaml b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_mmci_tl_test_224.yaml new file mode 100644 index 00000000..83cfd6b6 --- /dev/null +++ b/configs/experiment/predict/images/resnet50/multi_aggregator_eval/stain_mmci_tl_test_224.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/resnet50/multi_aggregator_eval/stain_base_224 + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/base.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/base.yaml new file mode 100644 index 00000000..2dfca5b9 --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/vgg16/base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 1 + extent_tile: 512 + stride_tile: 256 +top_k_aggregator: + kernel_size: 1 + extent_tile: 512 + stride_tile: 256 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/base_224.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/base_224.yaml new file mode 100644 index 00000000..38fff8c1 --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/base_224.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/vgg16/base_224 + - _self_ + +mean_pool_max_aggregator: + kernel_size: 7 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 5 + extent_tile: 224 + stride_tile: 112 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/mmci_tl_test.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/mmci_tl_test.yaml new file mode 100644 index 00000000..0e585312 --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/vgg16/multi_aggregator_eval/base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/mmci_tl_test_224.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/mmci_tl_test_224.yaml new file mode 100644 index 00000000..33168aef --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/mmci_tl_test_224.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/vgg16/multi_aggregator_eval/base_224 + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_base.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_base.yaml new file mode 100644 index 00000000..828285c8 --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_base.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/vgg16/stain_base + - _self_ + +mean_pool_max_aggregator: + kernel_size: 1 + extent_tile: 512 + stride_tile: 256 +top_k_aggregator: + kernel_size: 1 + extent_tile: 512 + stride_tile: 256 + k: 2 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_base_224.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_base_224.yaml new file mode 100644 index 00000000..89b12fe6 --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_base_224.yaml @@ -0,0 +1,21 @@ +# @package _global_ + +defaults: + - /callbacks/multi_aggregator_eval@callbacks + - /aggregator/max@max_aggregator + - /aggregator/mean_pool_max@mean_pool_max_aggregator + - /aggregator/top_k@top_k_aggregator + - /experiment/predict/images/vgg16/stain_base_224 + - _self_ + +mean_pool_max_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 +top_k_aggregator: + kernel_size: 1 + extent_tile: 224 + stride_tile: 112 + k: 4 + +use_case: "Multi-Aggregator SL Evaluation" diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_mmci_tl_test.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_mmci_tl_test.yaml new file mode 100644 index 00000000..f4c042ec --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_mmci_tl_test.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/vgg16/multi_aggregator_eval/stain_base + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_mmci_tl_test_224.yaml b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_mmci_tl_test_224.yaml new file mode 100644 index 00000000..ac6bfa41 --- /dev/null +++ b/configs/experiment/predict/images/vgg16/multi_aggregator_eval/stain_mmci_tl_test_224.yaml @@ -0,0 +1,6 @@ +# @package _global_ + +defaults: + - /experiment/predict/images/vgg16/multi_aggregator_eval/stain_base_224 + - /data/mmci/tl_test@predict_data + - _self_ diff --git a/preprocessing/embeddings/tile_embeddings.py b/preprocessing/embeddings/tile_embeddings.py index 1ba54a2f..b79dc315 100644 --- a/preprocessing/embeddings/tile_embeddings.py +++ b/preprocessing/embeddings/tile_embeddings.py @@ -86,7 +86,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: slide_embeddings[start:end] = embeddings save_embeddings(slide_embeddings, partition, slide_name, dest) - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"{e} occured during processing {slide_name}") logger.log_artifacts(local_dir=config.output_path) diff --git a/preprocessing/masks/quality_control_v2.py b/preprocessing/masks/quality_control_v2.py index e97d1cee..6282b13a 100644 --- a/preprocessing/masks/quality_control_v2.py +++ b/preprocessing/masks/quality_control_v2.py @@ -70,7 +70,7 @@ async def qc_main( total=len(slides), ): if not result.success: - with open(output_path / "qc_errors.log", "a") as log_file: + with open(output_path / "qc_errors.log", "a") as log_file: # noqa: ASYNC230 log_file.write( f"Failed to process {result.wsi_path}: {result.error}\n" ) diff --git a/prostate_cancer/callbacks/__init__.py b/prostate_cancer/callbacks/__init__.py index b2b3d3e8..60089480 100644 --- a/prostate_cancer/callbacks/__init__.py +++ b/prostate_cancer/callbacks/__init__.py @@ -9,6 +9,9 @@ ) from prostate_cancer.callbacks.heatmap_callback import HeatmapCallback from prostate_cancer.callbacks.mil_prediction_callback import MILPredictionCallback +from prostate_cancer.callbacks.multi_aggregator_eval_callback import ( + MultiAggregatorEvalCallback, +) from prostate_cancer.callbacks.nested_metrics_callback import NestedMetricsCallback from prostate_cancer.callbacks.num_positive_callback import NumPositiveCallback from prostate_cancer.callbacks.tile_histograms_callback import TileHistogramsCallback @@ -22,6 +25,7 @@ "EstimationCallback", "HeatmapCallback", "MILPredictionCallback", + "MultiAggregatorEvalCallback", "NestedMetricsCallback", "NumPositiveCallback", "TileHistogramsCallback", diff --git a/prostate_cancer/callbacks/cam_callback.py b/prostate_cancer/callbacks/cam_callback.py index 150b08cd..e0e482f0 100644 --- a/prostate_cancer/callbacks/cam_callback.py +++ b/prostate_cancer/callbacks/cam_callback.py @@ -29,7 +29,7 @@ def on_test_start( self, trainer: lightning.Trainer, pl_module: lightning.LightningModule ) -> None: if not isinstance(pl_module, CNNProstateModel): - raise ValueError("Model must be a CNNProstateModel to generate CAMs.") + raise TypeError("Model must be a CNNProstateModel to generate CAMs.") self.model = pl_module self.decode_head = cast("BinaryClassifier", self.model.decode_head) diff --git a/prostate_cancer/callbacks/multi_aggregator_eval_callback.py b/prostate_cancer/callbacks/multi_aggregator_eval_callback.py new file mode 100644 index 00000000..4507b88f --- /dev/null +++ b/prostate_cancer/callbacks/multi_aggregator_eval_callback.py @@ -0,0 +1,190 @@ +from copy import deepcopy +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +import lightning.pytorch as pl +import mlflow +import pandas as pd +import torch +from rationai.mlkit.lightning.callbacks import MultiloaderLifecycle +from rationai.mlkit.metrics.aggregators import Aggregator +from torchmetrics import ( + AUROC, + Accuracy, + NegativePredictiveValue, + Precision, + Recall, + Specificity, +) + +from prostate_cancer.typing import TilingSlideMetadata, UnlabeledTileSampleBatch + + +if TYPE_CHECKING: + from prostate_cancer.datamodule import TileDataModule + + +class MultiAggregatorEvalCallback(MultiloaderLifecycle): + """Aggregates TL predictions into SL predictions with the max, mean_pool_max and top_k aggregators.""" + + def __init__( + self, + max_aggregator: Aggregator, + mean_pool_max_aggregator: Aggregator, + top_k_aggregator: Aggregator, + max_threshold: float, + mean_pool_max_threshold: float, + top_k_threshold: float, + ) -> None: + super().__init__() + self.aggregators_original = { + "max": max_aggregator, + "mean_pool_max": mean_pool_max_aggregator, + "top_k": top_k_aggregator, + } + self.thresholds = { + "max": max_threshold, + "mean_pool_max": mean_pool_max_threshold, + "top_k": top_k_threshold, + } + + def setup( + self, + trainer: pl.Trainer, + pl_module: pl.LightningModule, + stage: str | None = None, + ) -> None: + self.tables: dict[str, dict[str, Any]] = { + name: { + "slide_name": [], + "prediction": [], + "prediction_binary": [], + "target": [], + } + for name in self.aggregators_original + } + self.majority_table: dict[str, Any] = { + "slide_name": [], + "prediction": [], + "target": [], + } + + def on_predict_dataloader_start( + self, trainer: pl.Trainer, pl_module: pl.LightningModule, dataloader_idx: int + ) -> None: + if not hasattr(trainer, "datamodule"): + raise ValueError("Trainer should have datamodule attribute") + # aggregators cannot be reset, thus their original state is copied for each slide + self.aggregators = { + name: deepcopy(aggregator) + for name, aggregator in self.aggregators_original.items() + } + datamodule = cast("TileDataModule", trainer.datamodule) + self.slide = cast( + "TilingSlideMetadata", datamodule.predict.slides[dataloader_idx] + ) + + def on_predict_batch_end( + self, + trainer: pl.Trainer, + pl_module: pl.LightningModule, + outputs: torch.Tensor, + batch: UnlabeledTileSampleBatch, + batch_idx: int, + dataloader_idx: int = 0, + ) -> None: + _, metadata = batch + + targets = torch.zeros_like(outputs) + for aggregator in self.aggregators.values(): + aggregator.update( + preds=outputs, + targets=targets, + x=metadata["x"], + y=metadata["y"], + ) + + def on_predict_dataloader_end( + self, trainer: pl.Trainer, pl_module: pl.LightningModule, dataloader_idx: int + ) -> None: + slide_name = Path(self.slide["path"]).stem + target = self.slide.get("carcinoma", None) + + votes = [] + for name, aggregator in self.aggregators.items(): + pred, _ = aggregator.compute() + pred_binary = bool(pred.item() >= self.thresholds[name]) + votes.append(pred_binary) + + table = self.tables[name] + table["slide_name"].append(slide_name) + table["prediction"].append(pred.item()) + table["prediction_binary"].append(pred_binary) + table["target"].append(target) + + self.majority_table["slide_name"].append(slide_name) + self.majority_table["prediction"].append(sum(votes) >= 2) + self.majority_table["target"].append(target) + + def on_predict_epoch_end( + self, trainer: pl.Trainer, pl_module: pl.LightningModule + ) -> None: + super().on_predict_epoch_end(trainer, pl_module) + + metrics: dict[str, float] = {} + + for name, table in self.tables.items(): + df = pd.DataFrame(table) + filename = f"sl_predictions_{name}.json" + df.to_json(filename, orient="split") + mlflow.log_artifact(filename, artifact_path="tables") + metrics.update( + self._compute_metrics( + name, + table["target"], + table["prediction_binary"], + table["prediction"], + ) + ) + + majority_df = pd.DataFrame(self.majority_table) + majority_df.to_json("sl_predictions_majority_vote.json", orient="split") + mlflow.log_artifact("sl_predictions_majority_vote.json", artifact_path="tables") + metrics.update( + self._compute_metrics( + "majority_vote", + self.majority_table["target"], + self.majority_table["prediction"], + score=None, + ) + ) + + mlflow.log_metrics(metrics) + + @staticmethod + def _compute_metrics( + name: str, + target: list[Any], + prediction_binary: list[bool], + score: list[float] | None = None, + ) -> dict[str, float]: + target_t = torch.tensor(target, dtype=torch.long) + pred_binary_t = torch.tensor(prediction_binary, dtype=torch.long) + + binary_metrics = { + "accuracy": Accuracy("binary"), + "precision": Precision("binary"), + "recall": Recall("binary"), + "specificity": Specificity("binary"), + "negative_predictive_value": NegativePredictiveValue("binary"), + } + results = { + f"sl/{name}/{metric_name}": metric(pred_binary_t, target_t).item() + for metric_name, metric in binary_metrics.items() + } + + if score is not None: + score_t = torch.tensor(score, dtype=torch.float) + results[f"sl/{name}/AUC"] = AUROC("binary")(score_t, target_t).item() + + return results diff --git a/prostate_cancer/datamodule/datasets/base.py b/prostate_cancer/datamodule/datasets/base.py index 4f91dc6f..8939d598 100644 --- a/prostate_cancer/datamodule/datasets/base.py +++ b/prostate_cancer/datamodule/datasets/base.py @@ -15,7 +15,7 @@ ) -T = TypeVar("T", covariant=True) +T_co = TypeVar("T_co", covariant=True) def get_slide_name(slide_metadata: TilingSlideMetadata) -> str: @@ -39,7 +39,7 @@ def __init__( ) -class BaseTileDataset(MetaTiledSlides[T]): +class BaseTileDataset(MetaTiledSlides[T_co]): """This class abstracts the functionality shared across embedding and image datasets.""" def __init__( @@ -75,7 +75,7 @@ def filter_non_carcinoma(self, tiles: HFDataset) -> HFDataset: ) ) - def generate_datasets(self) -> Iterable[Dataset[T]]: + def generate_datasets(self) -> Iterable[Dataset[T_co]]: tiles = self.tiles if self.labeled: @@ -95,7 +95,7 @@ def generate_datasets(self) -> Iterable[Dataset[T]]: return ( cast( - "Dataset[T]", + "Dataset[T_co]", self.single_slide_ds_cls( slide, tiles=self._meta.filter_tiles_by_slide(slide["id"]), diff --git a/prostate_cancer/datamodule/datasets/embeddings_dataset.py b/prostate_cancer/datamodule/datasets/embeddings_dataset.py index 61c95bcd..88156cd0 100644 --- a/prostate_cancer/datamodule/datasets/embeddings_dataset.py +++ b/prostate_cancer/datamodule/datasets/embeddings_dataset.py @@ -17,10 +17,10 @@ ) -T = TypeVar("T", covariant=True) +T_co = TypeVar("T_co", covariant=True) -class EmbeddingsDataset(BaseTileDataset[T]): +class EmbeddingsDataset(BaseTileDataset[T_co]): def __init__( self, uris: Iterable[str], diff --git a/prostate_cancer/datamodule/datasets/tile_dataset.py b/prostate_cancer/datamodule/datasets/tile_dataset.py index c7910a56..69bc42ab 100644 --- a/prostate_cancer/datamodule/datasets/tile_dataset.py +++ b/prostate_cancer/datamodule/datasets/tile_dataset.py @@ -19,10 +19,10 @@ ) -T = TypeVar("T", covariant=True) +T_co = TypeVar("T_co", covariant=True) -class TilesDataset(BaseTileDataset[T]): +class TilesDataset(BaseTileDataset[T_co]): def __init__( self, uris: Iterable[str],