diff --git a/algorithm_catalog/vito/worldcereal_crop_extent/records/worldcereal_crop_extent.json b/algorithm_catalog/vito/worldcereal_crop_extent/records/worldcereal_crop_extent.json index fd781a8ca..0596357e4 100644 --- a/algorithm_catalog/vito/worldcereal_crop_extent/records/worldcereal_crop_extent.json +++ b/algorithm_catalog/vito/worldcereal_crop_extent/records/worldcereal_crop_extent.json @@ -89,7 +89,14 @@ "name": "GeoTIFF" } ], - "license": "other" + "license": "other", + "validation": { + "document": { + "href": "https://doi.org/10.1038/s43016-023-00841-7", + "reviewed_by": "ESA", + "review_date": "2025-02-17" + } + } }, "linkTemplates": [], "links": [ diff --git a/algorithm_catalog/vito/worldcereal_crop_type/records/worldcereal_crop_type.json b/algorithm_catalog/vito/worldcereal_crop_type/records/worldcereal_crop_type.json index 99d11573a..e0b2d5c82 100644 --- a/algorithm_catalog/vito/worldcereal_crop_type/records/worldcereal_crop_type.json +++ b/algorithm_catalog/vito/worldcereal_crop_type/records/worldcereal_crop_type.json @@ -87,7 +87,14 @@ "name": "GeoTIFF" } ], - "license": "other" + "license": "other", + "validation": { + "document": { + "href": "https://doi.org/10.1038/s43016-023-00841-7", + "reviewed_by": "ESA", + "review_date": "2025-01-28" + } + } }, "linkTemplates": [], "links": [ diff --git a/docs/catalogue.md b/docs/catalogue.md index d232dbe36..c09a9576f 100644 --- a/docs/catalogue.md +++ b/docs/catalogue.md @@ -57,6 +57,43 @@ The following sections demonstrate how the various sections from the record are In those cases where the service is not intended to be publicly available, the `properties.visibility` property in the record can be set to `private`. This ensures that the service will not be listed in the [APEx Algorithm Catalogue](https://algorithm-catalogue.apex.esa.int/). The default value for this property is `public`, meaning that if it is not specified, the service will be visible in the catalogue. +## Validation Metadata and Certification + +Certification in the catalogue is derived from optional validation metadata in the service record. If this field is omitted, the service is treated as **Contributed** by default. + +Use `properties.validation` when a validation document exists: + +```json +"validation": { + "document": { + "href": "https://.../validation-report.pdf" + } +} +``` + +When ESA has reviewed and accepted the validation document, include ESA review metadata: + +```json +"validation": { + "document": { + "href": "https://.../validation-report.pdf", + "reviewed_by": "ESA", + "review_date": "2026-05-01" + } +} +``` + +A service can only be considered **ESA-validated** when: + +1. `properties.validation.document.reviewed_by = "ESA"` with `document.review_date`, and +2. at least one benchmark scenario JSON file exists in the service's `benchmark_scenarios/` folder. + +Suggested review flow: + +1. Algorithm PI adds the validation document URL in `validation.document.href`. +2. ESA reviewer verifies the document and benchmark readiness. +3. ESA reviewer updates the record with `reviewed_by` and `review_date`. + ## Services Overview @tbl-overview-mapping illustrates how the various sections from the record are connected to the general overview of all the services in the APEx Service Catalogue as shown in @fig-overview. diff --git a/qa/unittests/tests/test_records.py b/qa/unittests/tests/test_records.py index 6cf3f4bd9..9abf7a48f 100644 --- a/qa/unittests/tests/test_records.py +++ b/qa/unittests/tests/test_records.py @@ -2,6 +2,7 @@ import requests import pytest +from copy import deepcopy from apex_algorithm_qa_tools.common import get_project_root from pathlib import Path from apex_algorithm_qa_tools.records import ( @@ -86,3 +87,53 @@ def test_algorithm_provider_records_(): ) def test_provider_record_validation(record): jsonschema.validate(instance=record["data"], schema=get_provider_ogc_record_schema()) + + +def _has_benchmark_scenarios(record_path: str) -> bool: + service_dir = Path(record_path).parent.parent + benchmark_dir = service_dir / "benchmark_scenarios" + return benchmark_dir.exists() and any(benchmark_dir.glob("*.json")) + + +@pytest.mark.parametrize( + "record", + [ + pytest.param(record, id=record["data"]["id"]) + for record in get_service_ogc_records() + if record["data"] + .get("properties", {}) + .get("validation", {}) + .get("document", {}) + .get("reviewed_by") + == "ESA" + ], +) +def test_service_record_with_esa_reviewed_validation_requires_benchmark_scenarios(record): + assert _has_benchmark_scenarios(record["path"]), ( + f"Service '{record['data']['id']}' has ESA-reviewed validation metadata but no benchmark scenario files in " + f"{Path(record['path']).parent.parent / 'benchmark_scenarios'}" + ) + + +@pytest.mark.parametrize( + "invalid_validation", + [ + { + "document": { + "href": "https://example.com/validation-report.pdf", + "reviewed_by": "Other", + "review_date": "2025-01-01", + }, + }, + {"document": {"href": "https://example.com/validation-report.pdf", "reviewed_by": "ESA"}}, + {"document": {"href": "https://example.com/validation-report.pdf", "review_date": "2025-01-01"}}, + ], +) +def test_service_record_validation_rejects_malformed_validation_metadata( + invalid_validation, +): + record = deepcopy(get_service_ogc_records()[0]["data"]) + record["properties"]["validation"] = invalid_validation + + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(instance=record, schema=get_service_ogc_record_schema()) diff --git a/schemas/record.json b/schemas/record.json index 18a82bb63..3bd3e903b 100644 --- a/schemas/record.json +++ b/schemas/record.json @@ -427,6 +427,49 @@ "license": { "type": "string", "description": "License that is applicable for the APEx service." + }, + "validation": { + "type": "object", + "description": "Optional metadata describing the validation document for this service.", + "required": [ + "document" + ], + "properties": { + "document": { + "type": "object", + "description": "Reference to the validation document.", + "required": [ + "href" + ], + "properties": { + "href": { + "type": "string", + "format": "uri", + "description": "URL to the validation report or equivalent validation evidence." + }, + "reviewed_by": { + "type": "string", + "description": "Reviewer organization that accepted the validation document.", + "enum": [ + "ESA" + ] + }, + "review_date": { + "type": "string", + "format": "date", + "description": "Date on which the validation document was accepted." + } + }, + "dependentRequired": { + "reviewed_by": [ + "review_date" + ], + "review_date": [ + "reviewed_by" + ] + } + } + } } } },