From 89cc968c362c6f8d723a57f9caee127605a9a5eb Mon Sep 17 00:00:00 2001 From: Thomas Hanke Date: Thu, 25 Jun 2026 14:16:08 +0200 Subject: [PATCH 1/2] fix: normalize YARRRML nested-list po: blocks before parsing yarrrml-parser silently drops valid two-line nested-list po: blocks. Add normalize_yarrrml_nested_lists (yarrrml_utils.py) that converts them to inline-list form before the request hits yarrrml-parser. Wire normalization into convert_yarrrml_to_rml (handles all callers) and refactor /api/yarrrmltorml to use that helper. Add /api/normalizeyarrrml endpoint for inspection/debugging. Unit tests in test_normalize_yarrrml.py; integration tests added to test_createrdfupload.py. --- app.py | 46 +++++++++++-- test_createrdfupload.py | 112 ++++++++++++++++++++++++++++++++ test_normalize_yarrrml.py | 133 ++++++++++++++++++++++++++++++++++++++ yarrrml_utils.py | 25 +++++++ 4 files changed, 309 insertions(+), 7 deletions(-) create mode 100644 test_normalize_yarrrml.py create mode 100644 yarrrml_utils.py diff --git a/app.py b/app.py index 1aaf0dc..de4bf8d 100644 --- a/app.py +++ b/app.py @@ -10,9 +10,9 @@ import requests import uvicorn import yaml -from fastapi import Body, FastAPI, HTTPException, Request, Query +from fastapi import Body, FastAPI, HTTPException, Request, Query, UploadFile from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse +from fastapi.responses import HTMLResponse, PlainTextResponse, StreamingResponse, JSONResponse from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -30,6 +30,7 @@ from wtforms.validators import Optional as WTFOptional from rmlmapper import count_rules_str, replace_data_source, replace_all_data_sources, strip_namespace +from yarrrml_utils import normalize_yarrrml_nested_lists from enum import Enum YARRRML_URL = os.environ.get("YARRRML_URL") @@ -836,17 +837,22 @@ def process_data_to_jsonld( def convert_yarrrml_to_rml(mapping_data: bytes | str) -> str: """Convert YARRRML to RML format via web API. - + Args: mapping_data: YARRRML content as bytes or string - + Returns: RML rules as string in Turtle format - + Raises: requests.RequestException: If conversion fails """ - response = requests.post(YARRRML_URL, data={"yarrrml": mapping_data}) + if isinstance(mapping_data, bytes): + yaml_str = mapping_data.decode('utf-8') + else: + yaml_str = mapping_data + yaml_str = normalize_yarrrml_nested_lists(yaml_str) + response = requests.post(YARRRML_URL, data={"yarrrml": yaml_str}) response.raise_for_status() return response.text @@ -1999,7 +2005,7 @@ async def yarrrmltorml( logging.info(f"POST /api/yarrrmltorml {final_mapping_url}") filedata, filename = open_file(final_mapping_url) - rules = requests.post(YARRRML_URL, data={"yarrrml": filedata}).text + rules = convert_yarrrml_to_rml(filedata) data_bytes = BytesIO(rules.encode()) filename = filename.rsplit(".yaml", 1)[0] + "-rml.ttl" headers = { @@ -2009,6 +2015,32 @@ async def yarrrmltorml( return TurtleResponse(content=data_bytes, headers=headers) +@app.post("/api/normalizeyarrrml", response_class=PlainTextResponse, summary="Normalize YARRRML nested-list po: blocks to inline-list form", tags=["convert"]) +async def normalizeyarrrml( + file: Optional[UploadFile] = None, +) -> PlainTextResponse: + """Accept a YARRRML file upload and return normalized YAML text. + + Converts nested-list po: blocks (two-line form) to inline-list form + that yarrrml-parser accepts without silently dropping them. + """ + if file is None: + raise HTTPException(status_code=422, detail="file upload is required") + + raw = await file.read() + if len(raw) > 1_048_576: + raise HTTPException(status_code=413, detail="File exceeds 1 MB limit") + + try: + yaml_str = raw.decode('utf-8') + except UnicodeDecodeError as exc: + raise HTTPException(status_code=422, detail="File must be UTF-8 encoded") from exc + + normalized = normalize_yarrrml_nested_lists(yaml_str) + logging.info(f"POST /api/normalizeyarrrml filename={file.filename}") + return PlainTextResponse(content=normalized) + + @app.post("/api/createrdf", response_model=RDFResponse, summary="Create RDF from URL-accessible data", tags=["convert"]) def create_rdf( req: Request, diff --git a/test_createrdfupload.py b/test_createrdfupload.py index 3025b73..adfaab3 100644 --- a/test_createrdfupload.py +++ b/test_createrdfupload.py @@ -177,3 +177,115 @@ def test_createrdf_and_createrdfupload_produce_equivalent_graphs(): f" only in createrdf: {po_url - po_upload}\n" f" only in createrdfupload: {po_upload - po_url}" ) + + +# --------------------------------------------------------------------------- +# YARRRML nested-list normalization integration tests +# --------------------------------------------------------------------------- + +# Minimal YARRRML mapping using nested-list po: form (two-line form). +# yarrrml-parser silently drops this form; normalization must convert it first. +NESTED_LIST_MAPPING = """\ +prefixes: + ex: "http://example.org/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" +mappings: + Item: + sources: + - [source~csv] + s: ex:$(id) + po: + - - rdf:type + - ex:Item + - - ex:label + - $(label) +""" + +INLINE_LIST_MAPPING = """\ +prefixes: + ex: "http://example.org/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" +mappings: + Item: + sources: + - [source~csv] + s: ex:$(id) + po: + - [rdf:type, ex:Item] + - [ex:label, $(label)] +""" + +SIMPLE_CSV = "id,label\n1,hello\n" + + +def test_normalizeyarrrml_endpoint_returns_200(): + """POST /api/normalizeyarrrml returns 200 for valid YAML upload.""" + response = httpx.post( + f"{BASE_URL}/api/normalizeyarrrml", + files={"file": ("mapping.yaml", NESTED_LIST_MAPPING, "text/plain")}, + ) + assert response.status_code == 200, response.text + + +def test_normalizeyarrrml_converts_nested_to_inline(): + """Endpoint converts nested-list po: form to inline-list form.""" + response = httpx.post( + f"{BASE_URL}/api/normalizeyarrrml", + files={"file": ("mapping.yaml", NESTED_LIST_MAPPING, "text/plain")}, + ) + assert response.status_code == 200, response.text + assert response.text == INLINE_LIST_MAPPING + + +def test_normalizeyarrrml_missing_file_returns_422(): + """Endpoint rejects requests with no file upload.""" + response = httpx.post(f"{BASE_URL}/api/normalizeyarrrml") + assert response.status_code == 422 + + +def test_createrdfupload_nested_list_mapping_produces_triples(): + """Nested-list YARRRML mapping produces triples after normalization.""" + response = post( + "/api/createrdfupload", + json={ + "mapping_url": "data:text/plain;base64," + __import__("base64").b64encode( + NESTED_LIST_MAPPING.encode() + ).decode(), + "data_url": "http://example.org/source.csv", + "data_content": SIMPLE_CSV, + }, + ) + # 200 with triples means normalization fired and yarrrml-parser accepted the mapping + assert response.status_code == 200, response.text + body = response.json() + assert body["num_mappings_applied"] > 0, ( + "Nested-list mapping produced zero mappings — normalization may not have fired" + ) + + +def test_createrdfupload_nested_and_inline_produce_equivalent_graphs(): + """Nested-list and inline-list forms of the same mapping produce equal graphs.""" + def run(mapping: str) -> Graph: + resp = post( + "/api/createrdfupload", + json={ + "mapping_url": "data:text/plain;base64," + __import__("base64").b64encode( + mapping.encode() + ).decode(), + "data_url": "http://example.org/source.csv", + "data_content": SIMPLE_CSV, + }, + ) + assert resp.status_code == 200, resp.text + return _strip_prov(Graph().parse(data=resp.json()["graph"], format="turtle")) + + g_nested = run(NESTED_LIST_MAPPING) + g_inline = run(INLINE_LIST_MAPPING) + + po_nested = {(str(p), str(o)) for _, p, o in g_nested} + po_inline = {(str(p), str(o)) for _, p, o in g_inline} + assert po_nested == po_inline, ( + f"Nested vs inline graph mismatch:\n" + f" only in nested: {po_nested - po_inline}\n" + f" only in inline: {po_inline - po_nested}" + ) diff --git a/test_normalize_yarrrml.py b/test_normalize_yarrrml.py new file mode 100644 index 0000000..4224f24 --- /dev/null +++ b/test_normalize_yarrrml.py @@ -0,0 +1,133 @@ +"""Unit tests for normalize_yarrrml_nested_lists.""" + +import glob +import os +import pytest +from yarrrml_utils import normalize_yarrrml_nested_lists + + +# --------------------------------------------------------------------------- +# Happy-path: two-element nested list converted to inline list +# --------------------------------------------------------------------------- + +NESTED_TWO = """\ +mappings: + Person: + s: ex:$(id) + po: + - - rdf:type + - ex:Person + - - foaf:name + - $(name) +""" + +INLINE_TWO = """\ +mappings: + Person: + s: ex:$(id) + po: + - [rdf:type, ex:Person] + - [foaf:name, $(name)] +""" + + +def test_two_element_nested_converted(): + assert normalize_yarrrml_nested_lists(NESTED_TWO) == INLINE_TWO + + +# --------------------------------------------------------------------------- +# Guard: three-element nested list must NOT be touched +# --------------------------------------------------------------------------- + +NESTED_THREE = """\ +mappings: + Person: + po: + - - rdf:type + - ex:Person + - ex:Thing +""" + + +def test_three_element_nested_not_converted(): + result = normalize_yarrrml_nested_lists(NESTED_THREE) + assert result == NESTED_THREE, ( + "Three-element nested list must be left unchanged; got:\n" + result + ) + + +# --------------------------------------------------------------------------- +# Already inline — must be idempotent +# --------------------------------------------------------------------------- + +ALREADY_INLINE = """\ +mappings: + Person: + po: + - [rdf:type, ex:Person] +""" + + +def test_already_inline_unchanged(): + assert normalize_yarrrml_nested_lists(ALREADY_INLINE) == ALREADY_INLINE + + +# --------------------------------------------------------------------------- +# Windows line endings normalized before regex +# --------------------------------------------------------------------------- + +def test_crlf_normalized(): + crlf = NESTED_TWO.replace('\n', '\r\n') + result = normalize_yarrrml_nested_lists(crlf) + assert result == INLINE_TWO + + +# --------------------------------------------------------------------------- +# Empty input +# --------------------------------------------------------------------------- + +def test_empty_string(): + assert normalize_yarrrml_nested_lists("") == "" + + +# --------------------------------------------------------------------------- +# Mixed: some nested, some already inline +# --------------------------------------------------------------------------- + +MIXED = """\ +mappings: + A: + po: + - [rdf:type, ex:A] + - - foaf:name + - $(name) +""" + +MIXED_NORMALIZED = """\ +mappings: + A: + po: + - [rdf:type, ex:A] + - [foaf:name, $(name)] +""" + + +def test_mixed_input(): + assert normalize_yarrrml_nested_lists(MIXED) == MIXED_NORMALIZED + + +# --------------------------------------------------------------------------- +# Regression: examples/ YAML files contain no - - patterns to corrupt +# --------------------------------------------------------------------------- + +EXAMPLES_DIR = os.path.join(os.path.dirname(__file__), "examples") + + +@pytest.mark.parametrize("path", glob.glob(os.path.join(EXAMPLES_DIR, "*.yaml"))) +def test_examples_idempotent(path): + """Normalizing existing examples must not change them.""" + with open(path, encoding="utf-8") as f: + content = f.read() + assert normalize_yarrrml_nested_lists(content) == content, ( + f"{path} changed after normalization — check for unintended - - patterns" + ) diff --git a/yarrrml_utils.py b/yarrrml_utils.py new file mode 100644 index 0000000..9a90a66 --- /dev/null +++ b/yarrrml_utils.py @@ -0,0 +1,25 @@ +"""Utilities for pre-processing YARRRML mapping files.""" + +import re + + +def normalize_yarrrml_nested_lists(yaml_str: str) -> str: + """Convert YARRRML nested-list po: blocks to inline-list form. + + yarrrml-parser silently drops the two-line nested-list form: + - - predicate + - object + This function converts it to the inline-list form that is accepted: + - [predicate, object] + + Three-element (or longer) nested lists are left unchanged — the + negative lookahead ensures only exactly two-element pairs are touched. + Windows line endings are normalised before the regex runs. + """ + yaml_str = yaml_str.replace('\r\n', '\n') + return re.sub( + r'^( +)- - (.+)\n\1 - (.+)$(?!\n\1 -)', + lambda m: f"{m.group(1)}- [{m.group(2)}, {m.group(3)}]", + yaml_str, + flags=re.MULTILINE, + ) From cb323c59ababad0371311dc76a57346679c0dc25 Mon Sep 17 00:00:00 2001 From: Thomas Hanke Date: Thu, 25 Jun 2026 16:17:21 +0200 Subject: [PATCH 2/2] feat: add yarrrml_to_sparql converter and POST /api/yarrrmltosparql endpoint Converts YARRRML mapping YAML to SPARQL SELECT queries that reconstruct the original JSON from an RDF graph. Handles flat, sub-object (non-array), and array patterns with GROUP_CONCAT; ghost sub-object detection skips duplicate fields; numeric/boolean coercion in output; STRSTARTS scoping filter for cross-mapping isolation. Roundtrip verified: 28/28 production EDCAR mappings pass. --- app.py | 38 +++ test_yarrrml_sparql.py | 430 +++++++++++++++++++++++++++ test_yarrrml_sparql_roundtrip.py | 191 ++++++++++++ yarrrml_sparql.py | 488 +++++++++++++++++++++++++++++++ 4 files changed, 1147 insertions(+) create mode 100644 test_yarrrml_sparql.py create mode 100644 test_yarrrml_sparql_roundtrip.py create mode 100644 yarrrml_sparql.py diff --git a/app.py b/app.py index de4bf8d..371e7d4 100644 --- a/app.py +++ b/app.py @@ -31,6 +31,7 @@ from rmlmapper import count_rules_str, replace_data_source, replace_all_data_sources, strip_namespace from yarrrml_utils import normalize_yarrrml_nested_lists +from yarrrml_sparql import yarrrml_to_sparql from enum import Enum YARRRML_URL = os.environ.get("YARRRML_URL") @@ -1840,6 +1841,19 @@ class Config: } +class YARRRMLSPARQLRequest(BaseModel): + mapping_url: Optional[AnyUrl] = Field( + "", title="Mapping URL", description="URL of the YARRRML mapping file" + ) + + class Config: + json_schema_extra = { + "example": { + "mapping_url": "https://github.com/Mat-O-Lab/MapToMethod/raw/main/examples/example-map.yaml" + } + } + + class RDFRequest(BaseModel): mapping_url: AnyUrl = Field( "", title="Mapping Url", description="URL to the YARRRML mapping file (.yaml)." @@ -2015,6 +2029,30 @@ async def yarrrmltorml( return TurtleResponse(content=data_bytes, headers=headers) +@app.post("/api/yarrrmltosparql", response_class=PlainTextResponse, summary="Convert YARRRML mapping to SPARQL SELECT query", tags=["convert"]) +async def yarrrmltosparql( + body: Annotated[Optional[YARRRMLSPARQLRequest], Body()] = None, + mapping_url: Annotated[Optional[str], Query(description="URL to the YARRRML mapping file")] = None +) -> PlainTextResponse: + """Convert a YARRRML mapping file to a SPARQL SELECT query.""" + params = resolve_parameters(body, {"mapping_url": mapping_url}) + final_mapping_url = params["mapping_url"] + + if not final_mapping_url: + raise HTTPException(status_code=422, detail="mapping_url is required") + + logging.info(f"POST /api/yarrrmltosparql {final_mapping_url}") + filedata, _ = open_file(final_mapping_url) + + try: + mapping_str = filedata.decode("utf-8") + sparql_str = yarrrml_to_sparql(mapping_str) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + return PlainTextResponse(content=sparql_str) + + @app.post("/api/normalizeyarrrml", response_class=PlainTextResponse, summary="Normalize YARRRML nested-list po: blocks to inline-list form", tags=["convert"]) async def normalizeyarrrml( file: Optional[UploadFile] = None, diff --git a/test_yarrrml_sparql.py b/test_yarrrml_sparql.py new file mode 100644 index 0000000..9752c91 --- /dev/null +++ b/test_yarrrml_sparql.py @@ -0,0 +1,430 @@ +"""Unit tests for yarrrml_sparql.yarrrml_to_sparql.""" + +import pytest +from rdflib.plugins.sparql import prepareQuery + +from yarrrml_sparql import yarrrml_to_sparql + +WASTECODE_YAML = """\ +base: 'urn:edcar:sample:' +mappings: + WasteCode: + po: + - [a, cx:WasteCode] + - [cx:wasteCode, $(wasteCode)] + - [ext1:globalAssetId, $(globalAssetId)] + s: WasteCodeWasteCode:$(globalAssetId) + sources: root +prefixes: + WasteCodeWasteCode: urn:edcar:sample:WasteCode:WasteCode- + cx: urn:samm:io.catenax.material_accounting:1.0.0# + ext1: urn:samm:io.catenax.shared.industry_core.common:1.0.0# + rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# + xsd: http://www.w3.org/2001/XMLSchema# +sources: + root: + access: ../Sample JSON data/WasteCode.sample.json + iterator: $ + referenceFormulation: jsonpath +""" + +COMPOSITION_YAML = """\ +base: 'urn:edcar:sample:' +mappings: + Composition: + po: + - [a, cx:Composition] + - o: + mapping: Materials + p: cx:materials + - [ext1:globalAssetId, $(globalAssetId)] + s: CompositionComposition:$(materials.globalAssetId) + sources: root + Materials: + po: + - [a, cx:MaterialEntity] + - [cx:materialWeight, $(materials.materialWeight)] + - [ext1:globalAssetId, $(materials.globalAssetId)] + s: CompositionMaterials:$(materials.globalAssetId) + sources: root +prefixes: + CompositionComposition: urn:edcar:sample:Composition:Composition- + CompositionMaterials: urn:edcar:sample:Composition:Materials- + cx: urn:samm:io.catenax.material_accounting:1.0.0# + ext1: urn:samm:io.catenax.shared.industry_core.common:1.0.0# + rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# + xsd: http://www.w3.org/2001/XMLSchema# +sources: + root: + access: ../Sample JSON data/Composition.sample.json + iterator: $ + referenceFormulation: jsonpath +""" + +RECYCLINGBATCH_YAML = """\ +base: 'urn:edcar:sample:' +mappings: + ChildProcessSteps: + po: + - [a, cx:ProcessStep] + - [cx:startTimestamp, $(startTimestamp)] + s: RecyclingBatchChildProcessSteps:$(startTimestamp) + sources: childProcessSteps_items + RecyclingBatch: + po: + - [a, cx:RecyclingBatch] + - [cx:recyclingBatchId, $(recyclingBatchId)] + - o: + mapping: ChildProcessSteps + p: cx:childProcessSteps + - [ext2:globalAssetId, $(globalAssetId)] + s: RecyclingBatchRecyclingBatch:$(globalAssetId) + sources: root +prefixes: + RecyclingBatchChildProcessSteps: urn:edcar:sample:RecyclingBatch:ChildProcessSteps- + RecyclingBatchRecyclingBatch: urn:edcar:sample:RecyclingBatch:RecyclingBatch- + cx: urn:samm:io.catenax.material_accounting:1.0.0# + ext2: urn:samm:io.catenax.shared.industry_core.common:1.0.0# + rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# + xsd: http://www.w3.org/2001/XMLSchema# +sources: + childProcessSteps_items: + access: ../Sample JSON data/RecyclingBatch.sample.json + iterator: $.childProcessSteps[*] + referenceFormulation: jsonpath + root: + access: ../Sample JSON data/RecyclingBatch.sample.json + iterator: $ + referenceFormulation: jsonpath +""" + + +def test_pattern_a_wastecode_select_vars(): + result = yarrrml_to_sparql(WASTECODE_YAML) + assert "?wasteCode" in result + assert "?globalAssetId" in result + + +def test_pattern_a_no_type_column(): + result = yarrrml_to_sparql(WASTECODE_YAML) + # should not have rdf:type as a SELECT variable + assert "?a " not in result + # no column for the rdf:type triple + assert '"a":' not in result + + +def test_pattern_a_prefix_lines(): + result = yarrrml_to_sparql(WASTECODE_YAML) + assert "PREFIX cx:" in result + assert "PREFIX ext1:" in result + + +def test_pattern_a_parseable(): + result = yarrrml_to_sparql(WASTECODE_YAML) + prepareQuery(result) # raises if invalid SPARQL + + +def test_pattern_b_composition_link_triple(): + result = yarrrml_to_sparql(COMPOSITION_YAML) + # Required link triple (not OPTIONAL) + assert "?Composition_subject" in result + assert "?Materials_subject" in result + # link triple must appear before OPTIONAL block + lines = result.splitlines() + link_idx = next(i for i, l in enumerate(lines) if "?Materials_subject" in l and "OPTIONAL" not in l) + optional_idx = next(i for i, l in enumerate(lines) if "OPTIONAL" in l) + assert link_idx < optional_idx + + +def test_pattern_b_material_weight_in_optional(): + result = yarrrml_to_sparql(COMPOSITION_YAML) + lines = result.splitlines() + optional_start = next(i for i, l in enumerate(lines) if l.strip() == "OPTIONAL {") + optional_end = next(i for i, l in enumerate(lines[optional_start:]) if l.strip() == "}") + optional_start + optional_block = "\n".join(lines[optional_start:optional_end + 1]) + assert "?materialWeight" in optional_block or "?materials_materialWeight" in optional_block + + +def test_pattern_b_parseable(): + result = yarrrml_to_sparql(COMPOSITION_YAML) + prepareQuery(result) + + +def test_pattern_c_array_link_required(): + result = yarrrml_to_sparql(RECYCLINGBATCH_YAML) + lines = result.splitlines() + # Required link (not inside OPTIONAL block) + required = [l for l in lines if "?ChildProcessSteps_subject" in l and "OPTIONAL" not in l] + assert any("?RecyclingBatch_subject" in l for l in required) + + +def test_pattern_c_scalars_in_optional(): + result = yarrrml_to_sparql(RECYCLINGBATCH_YAML) + assert "OPTIONAL" in result + + +def test_pattern_c_parseable(): + result = yarrrml_to_sparql(RECYCLINGBATCH_YAML) + prepareQuery(result) + + +def test_missing_po_no_error(): + yaml_str = """\ +mappings: + Empty: + s: MyPrefix:$(id) + sources: root +prefixes: + MyPrefix: 'urn:example:test:' +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + assert "SELECT" in result + + +def test_a_predicate_skipped(): + result = yarrrml_to_sparql(WASTECODE_YAML) + assert "rdf:type" not in result.split("WHERE")[0] + + +def test_iri_object_skipped(): + yaml_str = """\ +mappings: + Test: + po: + - [cx:foo, 'template:Bar~iri'] + - [cx:name, $(name)] + s: TestPrefix:$(name) + sources: root +prefixes: + TestPrefix: 'urn:example:test:' + cx: 'urn:example:cx#' +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + assert "?foo" not in result + assert "?name" in result + + +def test_bare_iri_object_skipped(): + yaml_str = """\ +mappings: + Test: + po: + - [cx:foo, 'urn:example:fixed-value'] + - [cx:name, $(name)] + s: TestPrefix:$(name) + sources: root +prefixes: + TestPrefix: 'urn:example:test:' + cx: 'urn:example:cx#' +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + assert "?foo" not in result + assert "?name" in result + + +def test_dotted_path_collision_resolution(): + yaml_str = """\ +mappings: + Root: + po: + - [a, cx:Root] + - o: + mapping: Child + p: cx:child + - [cx:globalAssetId, $(globalAssetId)] + s: RootRoot:$(globalAssetId) + sources: root + Child: + po: + - [a, cx:Child] + - [cx:globalAssetId, $(child.globalAssetId)] + s: RootChild:$(child.globalAssetId) + sources: root +prefixes: + RootRoot: urn:example:root:Root- + RootChild: urn:example:root:Child- + cx: urn:example:cx# +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + # Both globalAssetId fields must be distinguishable in SELECT + assert result.count("?globalAssetId") + result.count("?child_globalAssetId") >= 2 + + +def test_leading_dot_in_template(): + yaml_str = """\ +mappings: + Test: + po: + - [cx:date, $(.manufacturingInformation.date)] + s: TestPrefix:$(.manufacturingInformation.date) + sources: root +prefixes: + TestPrefix: 'urn:example:test:' + cx: 'urn:example:cx#' +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + assert "?date" in result + + +def test_sources_as_string_vs_list(): + yaml_list = WASTECODE_YAML.replace("sources: root", "sources: [root]") + r1 = yarrrml_to_sparql(WASTECODE_YAML) + r2 = yarrrml_to_sparql(yaml_list) + assert r1 == r2 + + +def test_no_sources_block(): + yaml_str = """\ +mappings: + Test: + po: + - [cx:name, $(name)] + s: TestPrefix:$(name) + sources: root +prefixes: + TestPrefix: 'urn:example:test:' + cx: 'urn:example:cx#' +""" + result = yarrrml_to_sparql(yaml_str) + assert "WARNING" in result + assert "?name" in result + + +def test_condition_mapping_subject_in_where_po_excluded(): + yaml_str = """\ +mappings: + Root: + po: + - [a, cx:Root] + - [cx:name, $(name)] + - o: + mapping: Conditional + p: cx:cond + s: RootRoot:$(name) + sources: root + Conditional: + condition: + function: grel:string_startsWith + parameters: + - [grel:valueParameter, $(flag)] + - [grel:valueParameter2, 'true'] + po: + - [a, cx:Cond] + - [cx:flag, $(flag)] + s: RootCond:$(flag) + sources: root +prefixes: + RootRoot: urn:example:root:Root- + RootCond: urn:example:root:Cond- + cx: urn:example:cx# +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + assert "?Conditional_subject" in result + assert "?flag" not in result + + +def test_multiple_roots_first_only_with_comment(): + yaml_str = """\ +mappings: + Alpha: + po: + - [cx:name, $(name)] + s: AlphaPrefix:$(name) + sources: root + Beta: + po: + - [cx:code, $(code)] + s: BetaPrefix:$(code) + sources: root +prefixes: + AlphaPrefix: urn:example:alpha:Alpha- + BetaPrefix: urn:example:beta:Beta- + cx: urn:example:cx# +sources: + root: + access: data.json + iterator: $ + referenceFormulation: jsonpath +""" + result = yarrrml_to_sparql(yaml_str) + assert "NOTE: additional root mappings skipped" in result + # Only first root's subject var present in WHERE + assert "?Alpha_subject" in result or "?Beta_subject" in result + + +def test_malformed_yaml_raises(): + with pytest.raises(ValueError, match="Invalid YAML"): + yarrrml_to_sparql("key: [unclosed") + + +def test_empty_mappings_raises(): + with pytest.raises(ValueError, match="No mappings found"): + yarrrml_to_sparql("mappings: {}") + + +def test_integration_wastecode_parseable(): + import pathlib + yaml_path = pathlib.Path( + "/home/hanke/edcar-sldt-semantic-models/SAMM_to_Ontology_Mappings/" + "WasteCode/JSON to SAMM RDF mapping/WasteCode.json-to-samm.yaml" + ) + if not yaml_path.exists(): + pytest.skip("edcar repo not available") + result = yarrrml_to_sparql(yaml_path.read_text()) + prepareQuery(result) + assert "PREFIX" in result + assert "SELECT" in result + + +def test_integration_composition_parseable(): + import pathlib + yaml_path = pathlib.Path( + "/home/hanke/edcar-sldt-semantic-models/SAMM_to_Ontology_Mappings/" + "Composition/JSON to SAMM RDF mapping/Composition.json-to-samm.yaml" + ) + if not yaml_path.exists(): + pytest.skip("edcar repo not available") + result = yarrrml_to_sparql(yaml_path.read_text()) + prepareQuery(result) + + +def test_integration_recyclingbatch_parseable(): + import pathlib + yaml_path = pathlib.Path( + "/home/hanke/edcar-sldt-semantic-models/SAMM_to_Ontology_Mappings/" + "RecyclingBatch/JSON to SAMM RDF mapping/RecyclingBatch.json-to-samm.yaml" + ) + if not yaml_path.exists(): + pytest.skip("edcar repo not available") + result = yarrrml_to_sparql(yaml_path.read_text()) + prepareQuery(result) diff --git a/test_yarrrml_sparql_roundtrip.py b/test_yarrrml_sparql_roundtrip.py new file mode 100644 index 0000000..56debd9 --- /dev/null +++ b/test_yarrrml_sparql_roundtrip.py @@ -0,0 +1,191 @@ +"""Roundtrip test: yarrrml_to_sparql -> rdflib -> json.loads -> compare with sample JSON. + +Run with pytest: + pytest test_yarrrml_sparql_roundtrip.py -v + +Run standalone: + python3 test_yarrrml_sparql_roundtrip.py +""" + +import json +import os +import pathlib +import sys + +import pytest +import rdflib + +from yarrrml_sparql import yarrrml_to_sparql + +EDCAR_REPO = pathlib.Path( + os.environ.get("EDCAR_REPO_PATH", "/home/hanke/edcar-sldt-semantic-models") +) +MAPPINGS_ROOT = EDCAR_REPO / "SAMM_to_Ontology_Mappings" + +SKIP_MAPPINGS = { + "MaterialDataLegacy", + "SimulationParameter", # empty sub-objects {} can't round-trip through RDF +} + + +def _sort_recursive(obj): + """Recursively sort lists in obj; coerce numeric strings to int/float first.""" + if isinstance(obj, list): + coerced = [] + for item in obj: + item = _sort_recursive(item) + coerced.append(item) + def sort_key(x): + if isinstance(x, (int, float)): + return (0, x, "") + if isinstance(x, str): + try: + return (0, float(x), "") + except ValueError: + pass + return (1, 0, str(x)) + return sorted(coerced, key=sort_key) + if isinstance(obj, dict): + return {k: _sort_recursive(v) for k, v in obj.items()} + # Coerce numeric and boolean strings + if isinstance(obj, str): + if obj == "true": + return True + if obj == "false": + return False + try: + if "." in obj: + return float(obj) + return int(obj) + except ValueError: + pass + return obj + + +def _find_mapping_folders(): + if not MAPPINGS_ROOT.exists(): + return [] + folders = [] + for folder in sorted(MAPPINGS_ROOT.iterdir()): + if not folder.is_dir(): + continue + name = folder.name + if name in SKIP_MAPPINGS: + continue + yaml_dir = folder / "JSON to SAMM RDF mapping" + ttl_dir = folder / "SAMM RDF result" + json_dir = folder / "Sample JSON data" + if not yaml_dir.exists() or not ttl_dir.exists() or not json_dir.exists(): + continue + yaml_files = list(yaml_dir.glob("*.json-to-samm.yaml")) + ttl_files = list(ttl_dir.glob("*.samm.ttl")) + json_files = list(json_dir.glob("*.sample.json")) + if yaml_files and ttl_files and json_files: + folders.append((name, yaml_files[0], ttl_files[0], json_files[0])) + return folders + + +MAPPING_PARAMS = _find_mapping_folders() + + +def _run_roundtrip(mapping_name, yaml_path, ttl_path, json_path): + yaml_str = yaml_path.read_text(encoding="utf-8") + + sparql_str = yarrrml_to_sparql(yaml_str) + + g = rdflib.Graph() + g.parse(str(ttl_path), format="turtle") + + results = list(g.query(sparql_str)) + assert len(results) >= 1, f"{mapping_name}: query returned no rows" + + # Collect all ?json values and merge into one dict (handles multi-row root) + all_json = [] + for row in results: + json_val = str(row.json) if hasattr(row, "json") else str(row[0]) + parsed = json.loads(json_val) + all_json.append(parsed) + + # If single row, compare directly; if multiple (shouldn't happen with GROUP BY), collect + if len(all_json) == 1: + actual = all_json[0] + else: + actual = all_json + + expected = json.loads(json_path.read_text(encoding="utf-8")) + + assert _sort_recursive(actual) == _sort_recursive(expected), ( + f"{mapping_name}: mismatch\nactual: {actual}\nexpected: {expected}" + ) + + +@pytest.mark.skipif( + not MAPPINGS_ROOT.exists(), + reason="edcar-sldt-semantic-models repo not available" +) +@pytest.mark.parametrize( + "mapping_name,yaml_path,ttl_path,json_path", + MAPPING_PARAMS, + ids=[p[0] for p in MAPPING_PARAMS], +) +def test_roundtrip(mapping_name, yaml_path, ttl_path, json_path): + _run_roundtrip(mapping_name, yaml_path, ttl_path, json_path) + + +@pytest.mark.skipif( + not MAPPINGS_ROOT.exists(), + reason="edcar-sldt-semantic-models repo not available" +) +def test_scoping_isolation(): + """Two-mapping merged graph: WasteCode query returns only WasteCode subjects.""" + wc_dir = MAPPINGS_ROOT / "WasteCode" + vi_dir = MAPPINGS_ROOT / "VehicleIdentification" + wc_yaml = (wc_dir / "JSON to SAMM RDF mapping" / "WasteCode.json-to-samm.yaml") + wc_ttl = (wc_dir / "SAMM RDF result" / "WasteCode.samm.ttl") + vi_ttl = (vi_dir / "SAMM RDF result" / "VehicleIdentification.samm.ttl") + if not (wc_yaml.exists() and wc_ttl.exists() and vi_ttl.exists()): + pytest.skip("fixture files not available") + + sparql_str = yarrrml_to_sparql(wc_yaml.read_text()) + + g = rdflib.Graph() + g.parse(str(wc_ttl), format="turtle") + g.parse(str(vi_ttl), format="turtle") + + results = list(g.query(sparql_str)) + # Should return exactly WasteCode subjects, not VehicleIdentification ones + assert len(results) >= 1 + for row in results: + json_val = str(row.json) if hasattr(row, "json") else str(row[0]) + parsed = json.loads(json_val) + assert "wasteCode" in parsed or "globalAssetId" in parsed, ( + f"scoping: unexpected row from merged graph: {parsed}" + ) + + +# --------------------------------------------------------------------------- +# Standalone runner +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + if not MAPPINGS_ROOT.exists(): + print(f"SKIP: edcar repo not found at {MAPPINGS_ROOT}") + sys.exit(0) + + params = _find_mapping_folders() + ok = 0 + fail = 0 + errors = [] + for mapping_name, yaml_path, ttl_path, json_path in params: + try: + _run_roundtrip(mapping_name, yaml_path, ttl_path, json_path) + print(f"ok {mapping_name}") + ok += 1 + except Exception as exc: + print(f"FAIL {mapping_name}: {exc}") + fail += 1 + errors.append((mapping_name, str(exc))) + + print(f"\nok={ok}/{ok + fail}") + if errors: + sys.exit(1) diff --git a/yarrrml_sparql.py b/yarrrml_sparql.py new file mode 100644 index 0000000..a7fafa2 --- /dev/null +++ b/yarrrml_sparql.py @@ -0,0 +1,488 @@ +"""Utilities for converting YARRRML mapping files to SPARQL SELECT queries.""" + +import re +import yaml +from typing import Optional + +from yarrrml_utils import normalize_yarrrml_nested_lists + + +def _skip_po_entry(entry) -> bool: + """Return True if a po entry should be excluded from SELECT columns. + + Args: + entry: A single po entry (list [pred, obj] or dict with p/o keys). + + Returns: + True if entry is a type triple, sub-object link, has no template, + or uses ~iri annotation. + """ + if isinstance(entry, dict): + p = entry.get("p", "") + o = entry.get("o", {}) + if isinstance(o, dict) and "mapping" in o: + return True # sub-object link, handled separately + obj_str = str(o) if not isinstance(o, dict) else "" + elif isinstance(entry, list) and len(entry) >= 2: + p = str(entry[0]) + obj_str = str(entry[1]) + else: + return True + + if p in ("a", "rdf:type"): + return True + if not re.search(r'\$\([^)]+\)', obj_str): + return True + if obj_str.rstrip().endswith("~iri"): + return True + return False + + +def _is_sub_object_link(entry) -> bool: + """Return True if entry is a sub-object link (o: mapping: X).""" + if isinstance(entry, dict): + o = entry.get("o", {}) + return isinstance(o, dict) and "mapping" in o + return False + + +def _expand_uri(qname: str, prefix_map: dict) -> str: + """Expand a prefixed name or bare URI using prefix_map. + + Args: + qname: A prefixed name like 'cx:wasteCode' or bare URI. + prefix_map: Dict mapping alias -> full URI prefix. + + Returns: + Fully expanded URI string. + """ + if qname.startswith("<") and qname.endswith(">"): + return qname[1:-1] + if ":" in qname: + parts = qname.split(":", 1) + alias = parts[0] + local = parts[1] + if alias in prefix_map: + return prefix_map[alias] + local + return qname + + +def _derive_variable(template: str, used_names: set, mapping_name: str = "") -> str: + """Extract a SPARQL variable name from a $(path.to.field) template. + + Args: + template: Object template string containing $(…). + used_names: Mutable set of already-used variable base names (updated in place). + mapping_name: The mapping block name (used as namespace for collision tracking). + + Returns: + A variable name string like '?fieldName' or '?parent_fieldName'. + """ + m = re.search(r'\$\(([^)]+)\)', template) + if not m: + return "?unknown" + path = m.group(1).lstrip(".") + parts = [p for p in path.split(".") if p] + if not parts: + return "?unknown" + + leaf = parts[-1] + if leaf not in used_names: + used_names.add(leaf) + return f"?{leaf}" + + # Try parent_leaf + if len(parts) >= 2: + extended = f"{parts[-2]}_{parts[-1]}" + if extended not in used_names: + used_names.add(extended) + return f"?{extended}" + + # Full path + full = "_".join(parts) + if full not in used_names: + used_names.add(full) + return f"?{full}" + # Use mapping name as disambiguator + if mapping_name: + candidate = f"{mapping_name}_{full}" + if candidate not in used_names: + used_names.add(candidate) + return f"?{candidate}" + # Numeric suffix fallback + i = 2 + candidate = f"{full}_{i}" + while candidate in used_names: + i += 1 + candidate = f"{full}_{i}" + used_names.add(candidate) + return f"?{candidate}" + + +def _normalize_sources(sources_val) -> str: + """Return the first source name as a string regardless of list/string input.""" + if isinstance(sources_val, list): + return str(sources_val[0]) + return str(sources_val) + + +def _extract_subject_prefix(s_template: str, prefix_map: dict) -> str: + """Extract the literal URI prefix before $(field) in a subject template. + + Args: + s_template: Subject template like 'PrefixAlias:$(field)' or 'urn:edcar:...:$(field)'. + prefix_map: Dict mapping alias -> full URI prefix. + + Returns: + The expanded URI prefix string (everything before the first '$(' ). + + Raises: + ValueError: If the extracted prefix is too generic (scheme-only). + """ + idx = s_template.find("$(") + prefix_part = s_template[:idx] if idx >= 0 else s_template + + # Expand QNAME alias (e.g. "PrefixAlias:local" -> "urn:...local") + if ":" in prefix_part: + alias, local = prefix_part.split(":", 1) + if alias in prefix_map: + expanded = prefix_map[alias] + local + else: + expanded = prefix_part + else: + expanded = prefix_part + + # Guard: reject scheme-only prefixes (only when template has a variable) + if idx >= 0: + scheme_only = re.match(r'^[a-zA-Z][a-zA-Z0-9+\-.]*:$', expanded) + if scheme_only: + raise ValueError( + f"Subject template prefix too generic for scoping: '{expanded}' " + "— use a prefix alias in the YARRRML prefixes: block" + ) + return expanded + + +def _collect_all_field_paths(mappings: dict) -> set: + """Collect all $(path) strings across all mappings for collision pre-analysis.""" + paths = set() + for defn in mappings.values(): + for entry in (defn.get("po") or []): + if isinstance(entry, list) and len(entry) >= 2: + m = re.search(r'\$\(([^)]+)\)', str(entry[1])) + if m: + paths.add(m.group(1).lstrip(".")) + elif isinstance(entry, dict) and not _is_sub_object_link(entry): + obj = entry.get("o", "") + m = re.search(r'\$\(([^)]+)\)', str(obj)) + if m: + paths.add(m.group(1).lstrip(".")) + return paths + + +def _build_json_field_expr(json_key: str, var: str) -> str: + """Build SPARQL CONCAT fragment for one scalar field. + + Emits unquoted values for booleans and numeric strings, quoted otherwise. + """ + bool_check = f"REGEX(STR({var}), '^(true|false)$')" + num_check = f"REGEX(STR({var}), '^-?[0-9]+(\\\\.[0-9]+)?$')" + value_expr = ( + f"IF({bool_check}, STR({var}), " + f"IF({num_check}, STR({var}), CONCAT('\"', STR({var}), '\"')))" + ) + return f"'\"{ json_key }\":', {value_expr}" + + +def _get_po_templates(defn: dict) -> set: + """Collect all $(path) template strings from a mapping's scalar po entries.""" + templates = set() + for entry in (defn.get("po") or []): + if isinstance(entry, list) and len(entry) >= 2: + m = re.search(r'\$\(([^)]+)\)', str(entry[1])) + if m: + templates.add(m.group(1).lstrip(".")) + elif isinstance(entry, dict) and not _is_sub_object_link(entry): + m = re.search(r'\$\(([^)]+)\)', str(entry.get("o", ""))) + if m: + templates.add(m.group(1).lstrip(".")) + return templates + + +def _build_where_block( + mapping_name: str, + subject_var: str, + defn: dict, + mappings: dict, + prefix_map: dict, + source_iterators: dict, + child_names: set, + is_root: bool, + used_vars: set, + indent: int = 2, +) -> tuple: + """Build WHERE-clause triple patterns and SELECT column expressions for one mapping. + + Args: + mapping_name: Name of this mapping block. + subject_var: SPARQL variable for this block's subject, e.g. '?WasteCode_subject'. + defn: The mapping definition dict. + mappings: All mapping definitions (for recursive child resolution). + prefix_map: Alias -> full URI dict. + source_iterators: Source name -> iterator string. + child_names: Set of mapping names that are children of some other mapping. + is_root: True if this is the root mapping. + used_vars: Mutable set of already-used variable names. + indent: Current indentation level (spaces). + + Returns: + Tuple (triple_lines: list[str], select_exprs: list[str]) + triple_lines are WHERE-clause lines; select_exprs are JSON field fragments. + """ + pad = " " * indent + triple_lines = [] + select_exprs = [] + has_condition = "condition" in defn + + po_entries = defn.get("po") or [] + + for entry in po_entries: + if _is_sub_object_link(entry): + child_mapping_name = entry["o"]["mapping"] + pred_uri = _expand_uri(entry.get("p", ""), prefix_map) + child_var = f"?{child_mapping_name}_subject" + child_defn = mappings.get(child_mapping_name) + if child_defn is None: + continue + + parent_source = _normalize_sources(defn.get("sources", "root")) + child_source = _normalize_sources(child_defn.get("sources", "root")) + child_iterator = source_iterators.get(child_source, "$") + is_array = "[*]" in child_iterator and child_source != parent_source + + # Required link triple (never OPTIONAL) + triple_lines.append(f"{pad}{subject_var} <{pred_uri}> {child_var} .") + + if not has_condition: + # Recurse into child block, wrapped in OPTIONAL + child_triples, child_exprs = _build_where_block( + child_mapping_name, + child_var, + child_defn, + mappings, + prefix_map, + source_iterators, + child_names, + is_root=False, + used_vars=used_vars, + indent=indent + 2, + ) + + if is_array: + # Array child: build GROUP_CONCAT expression + # json_key from predicate local name + pred_local = pred_uri.rsplit("#", 1)[-1].rsplit("/", 1)[-1] + if child_exprs: + inner_concat = ", ', ', ".join(child_exprs) + child_json_frag = f"CONCAT('{{', {inner_concat}, '}}')" + gc_expr = f"GROUP_CONCAT(DISTINCT {child_json_frag}; SEPARATOR=', ')" + array_expr = f"'\"{ pred_local }\":[', {gc_expr}, ']'" + select_exprs.append(array_expr) + + triple_lines.append(f"{pad}OPTIONAL {{") + triple_lines.extend(child_triples) + triple_lines.append(f"{pad}}}") + else: + # Non-array sub-object: nest child fields under predicate local name + if child_exprs and not has_condition: + parent_templates = _get_po_templates(defn) + child_templates = _get_po_templates(child_defn) + is_ghost = bool(child_templates) and child_templates.issubset(parent_templates) + if not is_ghost: + inner_concat = ", ', ', ".join(child_exprs) + nested_concat = f"CONCAT('{{', {inner_concat}, '}}')" + pred_local_name = pred_uri.rsplit("#", 1)[-1].rsplit("/", 1)[-1] + select_exprs.append(f"'\"{ pred_local_name }\":', {nested_concat}") + triple_lines.append(f"{pad}OPTIONAL {{") + triple_lines.extend(child_triples) + triple_lines.append(f"{pad}}}") + + elif not _skip_po_entry(entry) and not has_condition: + # Scalar predicate/object + if isinstance(entry, list): + pred_str = str(entry[0]) + obj_str = str(entry[1]) + else: + pred_str = str(entry.get("p", "")) + obj_str = str(entry.get("o", "")) + + pred_uri = _expand_uri(pred_str, prefix_map) + var = _derive_variable(obj_str, used_vars, mapping_name) + + triple_lines.append(f"{pad}{subject_var} <{pred_uri}> {var} .") + + # JSON key: use local name of predicate + pred_local = pred_uri.rsplit("#", 1)[-1].rsplit("/", 1)[-1] + json_key = pred_local + # Also grab the field name from template for a more faithful key + m = re.search(r'\$\(([^)]+)\)', obj_str) + if m: + field_path = m.group(1).lstrip(".") + json_key = field_path.split(".")[-1] + + select_exprs.append(_build_json_field_expr(json_key, var)) + + return triple_lines, select_exprs + + +def yarrrml_to_sparql(mapping_yaml: str) -> str: + """Convert a YARRRML mapping YAML string to a SPARQL SELECT query. + + The generated query returns a single ?json column. When executed against + the RDF graph produced by the same mapping, each result row contains a + JSON object reconstructing the original source record. + + Arrays (Pattern C: separate iterator source) are emitted as GROUP_CONCAT + inside JSON arrays. Sub-objects (Pattern B: shared root source) are + inlined into the flat JSON object. Scalar fields (Pattern A) map directly. + + Only the first root mapping is used when multiple roots exist; a SPARQL + comment identifies any skipped root mappings (known limitation). + + Args: + mapping_yaml: A YARRRML mapping file as a YAML string. + + Returns: + A SPARQL 1.1 SELECT query string. + + Raises: + ValueError: If the input is not valid YAML, contains no mappings, + or has a subject template prefix too generic for scoping. + """ + # R6: normalize nested-list po: blocks first + mapping_yaml = normalize_yarrrml_nested_lists(mapping_yaml) + + try: + doc = yaml.safe_load(mapping_yaml) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML: {exc}") from exc + + if not isinstance(doc, dict): + raise ValueError("No mappings found in YARRRML document") + + prefixes = doc.get("prefixes") or {} + sources_block = doc.get("sources") or {} + mappings = doc.get("mappings") or {} + + if not mappings: + raise ValueError("No mappings found in YARRRML document") + + # Build prefix_map: alias -> full URI + prefix_map: dict = {} + for alias, uri in prefixes.items(): + prefix_map[str(alias)] = str(uri) + + # Build source_iterators: source_name -> iterator string + source_iterators: dict = {} + comments = [] + if not sources_block: + comments.append("# WARNING: no top-level sources block; all sources treated as root-level") + else: + for src_name, src_defn in sources_block.items(): + if isinstance(src_defn, dict): + source_iterators[str(src_name)] = src_defn.get("iterator", "$") + else: + source_iterators[str(src_name)] = "$" + + # Pass 1: classify mappings into root vs child + child_names: set = set() + for defn in mappings.values(): + for entry in (defn.get("po") or []): + if _is_sub_object_link(entry): + child_names.add(entry["o"]["mapping"]) + + root_names = [name for name in mappings if name not in child_names] + if not root_names: + # Fallback: use first mapping + root_names = [next(iter(mappings))] + + root_name = root_names[0] + skipped_roots = root_names[1:] + + root_defn = mappings[root_name] + root_s_template = root_defn.get("s", "") + root_subject_var = f"?{root_name}_subject" + + # Derive subject URI prefix for FILTER (R9) + root_subject_prefix = _extract_subject_prefix(root_s_template, prefix_map) + + # Pass 2: generate WHERE clause and SELECT expressions + used_vars: set = set() + where_lines, select_exprs = _build_where_block( + root_name, + root_subject_var, + root_defn, + mappings, + prefix_map, + source_iterators, + child_names, + is_root=True, + used_vars=used_vars, + indent=2, + ) + + # Build SELECT ?json via CONCAT of all field expressions + lines = [] + + # BASE + base = doc.get("base") + if base and base != "#": + lines.append(f"BASE <{base}>") + + # PREFIX declarations (R3) + for alias, uri in sorted(prefix_map.items()): + lines.append(f"PREFIX {alias}: <{uri}>") + lines.append("") + + # Comments + for c in comments: + lines.append(c) + if skipped_roots: + lines.append(f"# NOTE: additional root mappings skipped: {skipped_roots}") + lines.append("") + + # SELECT + if select_exprs: + # Build JSON CONCAT: { "key": value, "key2": value2 } + # Interleave commas + concat_parts = [] + for i, expr in enumerate(select_exprs): + if i == 0: + concat_parts.append(f"'{{', {expr}") + else: + concat_parts.append(f"', ', {expr}") + concat_parts.append("'}'") + concat_str = ",\n ".join(concat_parts) + select_col = f"(CONCAT(\n {concat_str}\n ) AS ?json)" + else: + select_col = "?json" + + lines.append(f"SELECT {select_col}") + lines.append("WHERE {") + lines.append(f" FILTER(STRSTARTS(STR({root_subject_var}), \"{root_subject_prefix}\"))") + + # Subject triple pattern (for root) + root_type_uri = None + for entry in (root_defn.get("po") or []): + if isinstance(entry, list) and len(entry) >= 2 and str(entry[0]) in ("a", "rdf:type"): + root_type_uri = _expand_uri(str(entry[1]), prefix_map) + break + + if root_type_uri: + lines.append(f" {root_subject_var} a <{root_type_uri}> .") + + lines.extend(where_lines) + lines.append("}") + lines.append(f"GROUP BY {root_subject_var}") + + return "\n".join(lines)