From 835346902ef8baefdfdd4400163d303ec1479375 Mon Sep 17 00:00:00 2001 From: Alvin Nahabwe Date: Sat, 18 Jul 2026 12:29:35 +0300 Subject: [PATCH] Add Grad-CAM explanation to the classification inference UI The Inference tab's image-classification panel gains an "Explain prediction (Grad-CAM heatmap)" checkbox. When ticked, the app calls the new /explain/image-classification endpoint instead of plain inference, shows the predicted class with its confidence, and renders the returned heatmap overlay so a non-expert user can see which regions drove the prediction. Verified with parse() on both changed files. Depends on the API-side endpoint (companion PR on no-code-transformers). Co-Authored-By: Claude Opus 4.8 --- server/deep_learning.R | 46 +++++++++++++++++++++++++++++++++--------- ui/deeplearning_ui.R | 7 +++++++ 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/server/deep_learning.R b/server/deep_learning.R index a2f88fe..8b85390 100644 --- a/server/deep_learning.R +++ b/server/deep_learning.R @@ -870,17 +870,34 @@ deep_learning = function() { observeEvent(input$start_img_class_inference, { req(input$infer_img_class_upload, input$infer_img_class_checkpoint_dropdown) - img_class_inference_result(list(status = "Running...", prediction = "Processing...", error = NULL)) + img_class_inference_result(list(status = "Running...", prediction = "Processing...", error = NULL, image_url = NULL)) tryCatch({ - req <- dl_request("/inference/image-classification") %>% - req_body_multipart( - image = curl::form_file(input$infer_img_class_upload$datapath), - model_checkpoint = input$infer_img_class_checkpoint_dropdown - ) - resp_data <- resp_body_json(req_perform(req)) - img_class_inference_result(list(status = "Success", prediction = resp_data$prediction, error = NULL)) + if (isTRUE(input$infer_img_class_explain)) { + # Grad-CAM: returns prediction, confidence, and a heatmap URL. + req <- dl_request("/explain/image-classification") %>% + req_body_multipart( + image = curl::form_file(input$infer_img_class_upload$datapath), + model_checkpoint = input$infer_img_class_checkpoint_dropdown + ) + resp_data <- resp_body_json(req_perform(req)) + pred_text <- resp_data$prediction + if (!is.null(resp_data$confidence)) { + pred_text <- sprintf("%s (confidence: %.1f%%)", pred_text, 100 * as.numeric(resp_data$confidence)) + } + img_class_inference_result(list(status = "Success", prediction = pred_text, + error = NULL, image_url = resp_data$output_url)) + } else { + req <- dl_request("/inference/image-classification") %>% + req_body_multipart( + image = curl::form_file(input$infer_img_class_upload$datapath), + model_checkpoint = input$infer_img_class_checkpoint_dropdown + ) + resp_data <- resp_body_json(req_perform(req)) + img_class_inference_result(list(status = "Success", prediction = resp_data$prediction, + error = NULL, image_url = NULL)) + } }, error = function(e) { - img_class_inference_result(list(status = "Error", prediction = NULL, error = as.character(e))) + img_class_inference_result(list(status = "Error", prediction = NULL, error = as.character(e), image_url = NULL)) }) }) @@ -930,7 +947,16 @@ deep_learning = function() { output$img_class_prediction_output <- renderText({ img_class_inference_result()$prediction }) - + # Grad-CAM heatmap overlay (only populated when "Explain" was requested). + output$img_class_explain_output <- renderImage({ + res <- img_class_inference_result() + req(res$status == "Success", res$image_url) + image_url <- paste0(api_url, res$image_url) + temp_file <- tempfile(fileext = ".png") + download.file(image_url, temp_file, mode = "wb") + list(src = temp_file, contentType = 'image/png', alt = "Grad-CAM heatmap") + }, deleteFile = TRUE) + output$seg_inference_status_ui <- renderUI({ res <- seg_inference_result() if (res$status == "Running...") { diff --git a/ui/deeplearning_ui.R b/ui/deeplearning_ui.R index 80837fd..a5cd040 100644 --- a/ui/deeplearning_ui.R +++ b/ui/deeplearning_ui.R @@ -359,6 +359,7 @@ deeplearning_ui = function() { textInput("infer_img_class_run_name", "Enter Run Name to Find Checkpoints", ""), selectInput("infer_img_class_checkpoint_dropdown", "Select Checkpoint", choices = NULL), fileInput("infer_img_class_upload", "Upload Image for Classification"), + checkboxInput("infer_img_class_explain", "Explain prediction (Grad-CAM heatmap)", value = FALSE), actionButton("start_img_class_inference", "Run Inference", class = "btn-info", style="margin-top: 10px;") ), hr(), @@ -367,6 +368,12 @@ deeplearning_ui = function() { div( style = "background-color: #f8f9fa; border: 1px solid #dee2e6; border-radius: 5px; padding: 15px; margin-top: 5px; min-height: 50px; font-size: 1.1em;", textOutput("img_class_prediction_output") + ), + conditionalPanel( + condition = "input.infer_img_class_explain == true", + h5("Why? Grad-CAM heatmap", style = "margin-top: 15px;"), + helpText("Warmer regions contributed more to the predicted class."), + imageOutput("img_class_explain_output", height = "auto") ) ),