From 89cc968c362c6f8d723a57f9caee127605a9a5eb Mon Sep 17 00:00:00 2001 From: Thomas Hanke Date: Thu, 25 Jun 2026 14:16:08 +0200 Subject: [PATCH] fix: normalize YARRRML nested-list po: blocks before parsing yarrrml-parser silently drops valid two-line nested-list po: blocks. Add normalize_yarrrml_nested_lists (yarrrml_utils.py) that converts them to inline-list form before the request hits yarrrml-parser. Wire normalization into convert_yarrrml_to_rml (handles all callers) and refactor /api/yarrrmltorml to use that helper. Add /api/normalizeyarrrml endpoint for inspection/debugging. Unit tests in test_normalize_yarrrml.py; integration tests added to test_createrdfupload.py. --- app.py | 46 +++++++++++-- test_createrdfupload.py | 112 ++++++++++++++++++++++++++++++++ test_normalize_yarrrml.py | 133 ++++++++++++++++++++++++++++++++++++++ yarrrml_utils.py | 25 +++++++ 4 files changed, 309 insertions(+), 7 deletions(-) create mode 100644 test_normalize_yarrrml.py create mode 100644 yarrrml_utils.py diff --git a/app.py b/app.py index 1aaf0dc..de4bf8d 100644 --- a/app.py +++ b/app.py @@ -10,9 +10,9 @@ import requests import uvicorn import yaml -from fastapi import Body, FastAPI, HTTPException, Request, Query +from fastapi import Body, FastAPI, HTTPException, Request, Query, UploadFile from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse +from fastapi.responses import HTMLResponse, PlainTextResponse, StreamingResponse, JSONResponse from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -30,6 +30,7 @@ from wtforms.validators import Optional as WTFOptional from rmlmapper import count_rules_str, replace_data_source, replace_all_data_sources, strip_namespace +from yarrrml_utils import normalize_yarrrml_nested_lists from enum import Enum YARRRML_URL = os.environ.get("YARRRML_URL") @@ -836,17 +837,22 @@ def process_data_to_jsonld( def convert_yarrrml_to_rml(mapping_data: bytes | str) -> str: """Convert YARRRML to RML format via web API. - + Args: mapping_data: YARRRML content as bytes or string - + Returns: RML rules as string in Turtle format - + Raises: requests.RequestException: If conversion fails """ - response = requests.post(YARRRML_URL, data={"yarrrml": mapping_data}) + if isinstance(mapping_data, bytes): + yaml_str = mapping_data.decode('utf-8') + else: + yaml_str = mapping_data + yaml_str = normalize_yarrrml_nested_lists(yaml_str) + response = requests.post(YARRRML_URL, data={"yarrrml": yaml_str}) response.raise_for_status() return response.text @@ -1999,7 +2005,7 @@ async def yarrrmltorml( logging.info(f"POST /api/yarrrmltorml {final_mapping_url}") filedata, filename = open_file(final_mapping_url) - rules = requests.post(YARRRML_URL, data={"yarrrml": filedata}).text + rules = convert_yarrrml_to_rml(filedata) data_bytes = BytesIO(rules.encode()) filename = filename.rsplit(".yaml", 1)[0] + "-rml.ttl" headers = { @@ -2009,6 +2015,32 @@ async def yarrrmltorml( return TurtleResponse(content=data_bytes, headers=headers) +@app.post("/api/normalizeyarrrml", response_class=PlainTextResponse, summary="Normalize YARRRML nested-list po: blocks to inline-list form", tags=["convert"]) +async def normalizeyarrrml( + file: Optional[UploadFile] = None, +) -> PlainTextResponse: + """Accept a YARRRML file upload and return normalized YAML text. + + Converts nested-list po: blocks (two-line form) to inline-list form + that yarrrml-parser accepts without silently dropping them. + """ + if file is None: + raise HTTPException(status_code=422, detail="file upload is required") + + raw = await file.read() + if len(raw) > 1_048_576: + raise HTTPException(status_code=413, detail="File exceeds 1 MB limit") + + try: + yaml_str = raw.decode('utf-8') + except UnicodeDecodeError as exc: + raise HTTPException(status_code=422, detail="File must be UTF-8 encoded") from exc + + normalized = normalize_yarrrml_nested_lists(yaml_str) + logging.info(f"POST /api/normalizeyarrrml filename={file.filename}") + return PlainTextResponse(content=normalized) + + @app.post("/api/createrdf", response_model=RDFResponse, summary="Create RDF from URL-accessible data", tags=["convert"]) def create_rdf( req: Request, diff --git a/test_createrdfupload.py b/test_createrdfupload.py index 3025b73..adfaab3 100644 --- a/test_createrdfupload.py +++ b/test_createrdfupload.py @@ -177,3 +177,115 @@ def test_createrdf_and_createrdfupload_produce_equivalent_graphs(): f" only in createrdf: {po_url - po_upload}\n" f" only in createrdfupload: {po_upload - po_url}" ) + + +# --------------------------------------------------------------------------- +# YARRRML nested-list normalization integration tests +# --------------------------------------------------------------------------- + +# Minimal YARRRML mapping using nested-list po: form (two-line form). +# yarrrml-parser silently drops this form; normalization must convert it first. +NESTED_LIST_MAPPING = """\ +prefixes: + ex: "http://example.org/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" +mappings: + Item: + sources: + - [source~csv] + s: ex:$(id) + po: + - - rdf:type + - ex:Item + - - ex:label + - $(label) +""" + +INLINE_LIST_MAPPING = """\ +prefixes: + ex: "http://example.org/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" +mappings: + Item: + sources: + - [source~csv] + s: ex:$(id) + po: + - [rdf:type, ex:Item] + - [ex:label, $(label)] +""" + +SIMPLE_CSV = "id,label\n1,hello\n" + + +def test_normalizeyarrrml_endpoint_returns_200(): + """POST /api/normalizeyarrrml returns 200 for valid YAML upload.""" + response = httpx.post( + f"{BASE_URL}/api/normalizeyarrrml", + files={"file": ("mapping.yaml", NESTED_LIST_MAPPING, "text/plain")}, + ) + assert response.status_code == 200, response.text + + +def test_normalizeyarrrml_converts_nested_to_inline(): + """Endpoint converts nested-list po: form to inline-list form.""" + response = httpx.post( + f"{BASE_URL}/api/normalizeyarrrml", + files={"file": ("mapping.yaml", NESTED_LIST_MAPPING, "text/plain")}, + ) + assert response.status_code == 200, response.text + assert response.text == INLINE_LIST_MAPPING + + +def test_normalizeyarrrml_missing_file_returns_422(): + """Endpoint rejects requests with no file upload.""" + response = httpx.post(f"{BASE_URL}/api/normalizeyarrrml") + assert response.status_code == 422 + + +def test_createrdfupload_nested_list_mapping_produces_triples(): + """Nested-list YARRRML mapping produces triples after normalization.""" + response = post( + "/api/createrdfupload", + json={ + "mapping_url": "data:text/plain;base64," + __import__("base64").b64encode( + NESTED_LIST_MAPPING.encode() + ).decode(), + "data_url": "http://example.org/source.csv", + "data_content": SIMPLE_CSV, + }, + ) + # 200 with triples means normalization fired and yarrrml-parser accepted the mapping + assert response.status_code == 200, response.text + body = response.json() + assert body["num_mappings_applied"] > 0, ( + "Nested-list mapping produced zero mappings — normalization may not have fired" + ) + + +def test_createrdfupload_nested_and_inline_produce_equivalent_graphs(): + """Nested-list and inline-list forms of the same mapping produce equal graphs.""" + def run(mapping: str) -> Graph: + resp = post( + "/api/createrdfupload", + json={ + "mapping_url": "data:text/plain;base64," + __import__("base64").b64encode( + mapping.encode() + ).decode(), + "data_url": "http://example.org/source.csv", + "data_content": SIMPLE_CSV, + }, + ) + assert resp.status_code == 200, resp.text + return _strip_prov(Graph().parse(data=resp.json()["graph"], format="turtle")) + + g_nested = run(NESTED_LIST_MAPPING) + g_inline = run(INLINE_LIST_MAPPING) + + po_nested = {(str(p), str(o)) for _, p, o in g_nested} + po_inline = {(str(p), str(o)) for _, p, o in g_inline} + assert po_nested == po_inline, ( + f"Nested vs inline graph mismatch:\n" + f" only in nested: {po_nested - po_inline}\n" + f" only in inline: {po_inline - po_nested}" + ) diff --git a/test_normalize_yarrrml.py b/test_normalize_yarrrml.py new file mode 100644 index 0000000..4224f24 --- /dev/null +++ b/test_normalize_yarrrml.py @@ -0,0 +1,133 @@ +"""Unit tests for normalize_yarrrml_nested_lists.""" + +import glob +import os +import pytest +from yarrrml_utils import normalize_yarrrml_nested_lists + + +# --------------------------------------------------------------------------- +# Happy-path: two-element nested list converted to inline list +# --------------------------------------------------------------------------- + +NESTED_TWO = """\ +mappings: + Person: + s: ex:$(id) + po: + - - rdf:type + - ex:Person + - - foaf:name + - $(name) +""" + +INLINE_TWO = """\ +mappings: + Person: + s: ex:$(id) + po: + - [rdf:type, ex:Person] + - [foaf:name, $(name)] +""" + + +def test_two_element_nested_converted(): + assert normalize_yarrrml_nested_lists(NESTED_TWO) == INLINE_TWO + + +# --------------------------------------------------------------------------- +# Guard: three-element nested list must NOT be touched +# --------------------------------------------------------------------------- + +NESTED_THREE = """\ +mappings: + Person: + po: + - - rdf:type + - ex:Person + - ex:Thing +""" + + +def test_three_element_nested_not_converted(): + result = normalize_yarrrml_nested_lists(NESTED_THREE) + assert result == NESTED_THREE, ( + "Three-element nested list must be left unchanged; got:\n" + result + ) + + +# --------------------------------------------------------------------------- +# Already inline — must be idempotent +# --------------------------------------------------------------------------- + +ALREADY_INLINE = """\ +mappings: + Person: + po: + - [rdf:type, ex:Person] +""" + + +def test_already_inline_unchanged(): + assert normalize_yarrrml_nested_lists(ALREADY_INLINE) == ALREADY_INLINE + + +# --------------------------------------------------------------------------- +# Windows line endings normalized before regex +# --------------------------------------------------------------------------- + +def test_crlf_normalized(): + crlf = NESTED_TWO.replace('\n', '\r\n') + result = normalize_yarrrml_nested_lists(crlf) + assert result == INLINE_TWO + + +# --------------------------------------------------------------------------- +# Empty input +# --------------------------------------------------------------------------- + +def test_empty_string(): + assert normalize_yarrrml_nested_lists("") == "" + + +# --------------------------------------------------------------------------- +# Mixed: some nested, some already inline +# --------------------------------------------------------------------------- + +MIXED = """\ +mappings: + A: + po: + - [rdf:type, ex:A] + - - foaf:name + - $(name) +""" + +MIXED_NORMALIZED = """\ +mappings: + A: + po: + - [rdf:type, ex:A] + - [foaf:name, $(name)] +""" + + +def test_mixed_input(): + assert normalize_yarrrml_nested_lists(MIXED) == MIXED_NORMALIZED + + +# --------------------------------------------------------------------------- +# Regression: examples/ YAML files contain no - - patterns to corrupt +# --------------------------------------------------------------------------- + +EXAMPLES_DIR = os.path.join(os.path.dirname(__file__), "examples") + + +@pytest.mark.parametrize("path", glob.glob(os.path.join(EXAMPLES_DIR, "*.yaml"))) +def test_examples_idempotent(path): + """Normalizing existing examples must not change them.""" + with open(path, encoding="utf-8") as f: + content = f.read() + assert normalize_yarrrml_nested_lists(content) == content, ( + f"{path} changed after normalization — check for unintended - - patterns" + ) diff --git a/yarrrml_utils.py b/yarrrml_utils.py new file mode 100644 index 0000000..9a90a66 --- /dev/null +++ b/yarrrml_utils.py @@ -0,0 +1,25 @@ +"""Utilities for pre-processing YARRRML mapping files.""" + +import re + + +def normalize_yarrrml_nested_lists(yaml_str: str) -> str: + """Convert YARRRML nested-list po: blocks to inline-list form. + + yarrrml-parser silently drops the two-line nested-list form: + - - predicate + - object + This function converts it to the inline-list form that is accepted: + - [predicate, object] + + Three-element (or longer) nested lists are left unchanged — the + negative lookahead ensures only exactly two-element pairs are touched. + Windows line endings are normalised before the regex runs. + """ + yaml_str = yaml_str.replace('\r\n', '\n') + return re.sub( + r'^( +)- - (.+)\n\1 - (.+)$(?!\n\1 -)', + lambda m: f"{m.group(1)}- [{m.group(2)}, {m.group(3)}]", + yaml_str, + flags=re.MULTILINE, + )