From 91a98f42fa779e0448960590fc45907df8190922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sun, 5 Jul 2026 09:23:51 +0200 Subject: [PATCH 1/4] mf --- eval.py | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 11 deletions(-) diff --git a/eval.py b/eval.py index 71bfa24..e1f63b4 100644 --- a/eval.py +++ b/eval.py @@ -3,24 +3,36 @@ import numpy as np import csv import glob +import json from datasets import DATASETS, get_fn, prepare, get_h5_item def get_all_results(dirname): - """Yield (path, attrs, knns) for every valid result .h5 file under dirname.""" + """Return all paths with result .h5 file under dirname.""" seen = set() mask = dirname + "/**/*.h5" print(f"Searching for results matching: {mask}") + ret = [] for fn in glob.iglob(mask, recursive=True): if fn in seen: continue seen.add(fn) - with h5py.File(fn, "r") as f: - if "knns" not in f or "dataset" not in f.attrs or "task" not in f.attrs: - print(f"Ignoring {fn}") - continue - print(fn) - yield fn, dict(f.attrs), np.array(f["knns"]) + try: + with h5py.File(fn, "r") as f: + if "knns" not in f: + print(f"Ignoring {fn}") + continue + + ret += [fn] + except: + print(f"Could not load {fn}") + + return ret + + +def load_results(fn): + with h5py.File(fn, "r") as f: + return dict(f.attrs), np.array(f["knns"]) def get_recall(I, gt, k): @@ -60,6 +72,38 @@ def get_recall(I, gt, k): return hits.sum() / (len(I) * k) +def add_details_from_tira(fn, row): + software_details = Path(fn).parent.parent / "execution-details.json" + + if software_details.is_file(): + software_details = json.loads(software_details.read_text())["system"] + to_delete = ["docker_software_id", "user_image_name", "tira_image_name", "cache_behaviour", "task_id", "paper_link", "input_docker_software", "link_code", "mount_hf_model", "workflow_configuration", "forward_environment_variable", "input_docker_software_id", "input_upload_id", "ir_re_ranker", "ir_re_ranking_input", "previous_stages", "tira_image_workdir"] + for i in to_delete: + del software_details[i] + software_details = json.dumps(software_details) + else: + software_details = None + + ir_metadata = Path(fn).parent / ".tracking-results.yml" + if ir_metadata.is_file(): + import yaml + ir_metadata = yaml.safe_load(ir_metadata.read_text()) + tmp = {} + for i in ["cpu", "ram"]: + for j in ["system", "process"]: + tmp[f"{i}-{j}-max"] = ir_metadata["resources"][i][f"used {j}"]["max"] + + tmp["process-wallclock"] = ir_metadata["resources"]["runtime"]["wallclock"] + ir_metadata = tmp + else: + ir_metadata = {} + + row["software"] = software_details + if ir_metadata: + for k, v in ir_metadata.items(): + row[k] = v + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( @@ -67,19 +111,49 @@ def get_recall(I, gt, k): help="directory in which results are stored", default="results", ) + parser.add_argument( + "--dataset", + help="the dataset to evaluate (otherwise look into the .h5 attributes)", + default=None, + ) + parser.add_argument( + "--task", + help="the task to evaluate (otherwise look into the .h5 attributes)", + default=None, + ) + parser.add_argument( + "--cache", + help="cache file for faster ", + default=None, + ) parser.add_argument("csvfile") args = parser.parse_args() gt_cache = {} # (dataset, task) -> gt_I array columns = ["dataset", "task", "algo", "buildtime", "querytime", "params", "recall"] + row_cache = {} + + if args.cache and Path(args.cache).is_file(): + with open(args.cache, "r") as f: + for l in f: + try: + l = json.loads(l) + row_cache[l["fn"]] = l["row"] + except: + pass + print(f"Have {len(row_cache)} lines in cache {args.cache}.") with open(args.csvfile, "w", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=columns, extrasaction="ignore") writer.writeheader() - for fn, attrs, knns in get_all_results(args.results): - dataset = attrs["dataset"] - task = attrs["task"] + for fn in tqdm(list(get_all_results(args.results))): + if fn in row_cache: + writer.writerow(row_cache[fn]) + continue + attrs, knns = load_results(fn) + dataset = attrs["dataset"] if not args.dataset else args.dataset + task = attrs["task"] if not args.task else args.task if dataset not in DATASETS or task not in DATASETS[dataset]: print(f"Skipping {fn}: unknown dataset={dataset!r} task={task!r}") @@ -100,5 +174,11 @@ def get_recall(I, gt, k): recall = get_recall(knns, gt_I, k) row = dict(attrs) row["recall"] = recall - print(dataset, task, attrs.get("algo"), attrs.get("params"), "=>", recall) + add_details_from_tira(fn, row) + writer.writerow(row) + + if args.cache: + with open(args.cache, "a+") as f: + f.write(json.dumps({"fn": fn, "row": row}) + "\n") + From 9b2dc9046d98a7f9f616d06dbccb95e3849c737a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sun, 5 Jul 2026 09:31:18 +0200 Subject: [PATCH 2/4] make file executable --- eval.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 eval.py diff --git a/eval.py b/eval.py old mode 100644 new mode 100755 index e1f63b4..17b7365 --- a/eval.py +++ b/eval.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import h5py import numpy as np From 238ccff9ceb5b4921016feca0c9de5f6bdc33f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sun, 5 Jul 2026 09:51:14 +0200 Subject: [PATCH 3/4] unify types when loading attributes --- eval.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/eval.py b/eval.py index 17b7365..940667b 100755 --- a/eval.py +++ b/eval.py @@ -5,6 +5,8 @@ import csv import glob import json +from pathlib import Path +from tqdm import tqdm from datasets import DATASETS, get_fn, prepare, get_h5_item @@ -33,7 +35,15 @@ def get_all_results(dirname): def load_results(fn): with h5py.File(fn, "r") as f: - return dict(f.attrs), np.array(f["knns"]) + attrs = dict(f.attrs) + for k, v in attrs.items(): + if isinstance(v, bytes): + attrs[k] = v.decode("UTF-8") + if isinstance(v, np.int64): + attrs[k] = int(v) + if isinstance(v, np.float32): + attrs[k] = float(v) + return attrs, np.array(f["knns"]) def get_recall(I, gt, k): @@ -49,6 +59,10 @@ def get_recall(I, gt, k): Always compares the k nearest neighbors EXCLUDING self-loops. """ + if I.shape[0] != gt.shape[0]: + print(I.shape[0]) + print(gt.shape[0]) + return -1 assert I.shape[0] == gt.shape[0], "query count mismatch between results and ground truth" assert gt.shape[1] >= k + 1, f"Ground truth needs at least {k+1} columns (self + {k} neighbors), got {gt.shape[1]}" From 3bf3e4b283b5e7b87ab6b055f6e390e65dd52aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sun, 5 Jul 2026 10:02:59 +0200 Subject: [PATCH 4/4] add explanation for recall score --- eval.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/eval.py b/eval.py index 940667b..3f6866a 100755 --- a/eval.py +++ b/eval.py @@ -60,22 +60,26 @@ def get_recall(I, gt, k): Always compares the k nearest neighbors EXCLUDING self-loops. """ if I.shape[0] != gt.shape[0]: - print(I.shape[0]) - print(gt.shape[0]) - return -1 + return (-1, f"Query count mismatch {I.shape[0]} vs. {gt.shape[0]}") + assert I.shape[0] == gt.shape[0], "query count mismatch between results and ground truth" assert gt.shape[1] >= k + 1, f"Ground truth needs at least {k+1} columns (self + {k} neighbors), got {gt.shape[1]}" + msg = None if I.shape[1] == k + 1: # Results include self-loops: use columns [1:k+1] (skip first column) I_to_compare = I[:, 1:k+1] - print(f" Results shape {I.shape}: assumed self-loops, comparing columns [1:{k+1}]") + msg = f"OK, Results shape {I.shape}: assumed self-loops, comparing columns [1:{k+1}]" + print(msg) elif I.shape[1] == k: # Results exclude self-loops: use columns [:k] I_to_compare = I[:, :k] - print(f" Results shape {I.shape}: assumed no self-loops, comparing columns [0:{k}]") + msg = f"Ok, Results shape {I.shape}: assumed no self-loops, comparing columns [0:{k}]" + print(msg) else: - raise ValueError(f"Results shape {I.shape} is neither {I.shape[0]}x{k} nor {I.shape[0]}x{k+1}") + msg = f"Results shape {I.shape} is neither {I.shape[0]}x{k} nor {I.shape[0]}x{k+1}" + print(msg) + return (-1, msg) # Ground truth: always skip first column (self-loop) and use next k columns gt_to_compare = gt[:, 1:k+1] # Skip self-loop, take next k neighbors @@ -84,7 +88,7 @@ def get_recall(I, gt, k): len(np.intersect1d(I_to_compare[i], gt_to_compare[i])) for i in range(len(I)) ]) - return hits.sum() / (len(I) * k) + return (hits.sum() / (len(I) * k), msg) def add_details_from_tira(fn, row): @@ -188,7 +192,8 @@ def add_details_from_tira(fn, row): k = DATASETS[dataset][task]["k"] recall = get_recall(knns, gt_I, k) row = dict(attrs) - row["recall"] = recall + row["recall"] = recall[0] + row["recall_description"] = recall[1] add_details_from_tira(fn, row) writer.writerow(row)