fix: normalize YARRRML nested-list po: blocks before parsing

app.py

@@ -10,9 +10,9 @@
 import requests
 import uvicorn
 import yaml
-from fastapi import Body, FastAPI, HTTPException, Request, Query
+from fastapi import Body, FastAPI, HTTPException, Request, Query, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse, StreamingResponse, JSONResponse
 from fastapi.encoders import jsonable_encoder
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
@@ -30,6 +30,7 @@
 from wtforms.validators import Optional as WTFOptional
 
 from rmlmapper import count_rules_str, replace_data_source, replace_all_data_sources, strip_namespace
+from yarrrml_utils import normalize_yarrrml_nested_lists
 from enum import Enum
 
 YARRRML_URL = os.environ.get("YARRRML_URL")
@@ -836,17 +837,22 @@ def process_data_to_jsonld(
 
 def convert_yarrrml_to_rml(mapping_data: bytes | str) -> str:
     """Convert YARRRML to RML format via web API.
-    
+
     Args:
         mapping_data: YARRRML content as bytes or string
-        
+
     Returns:
         RML rules as string in Turtle format
-        
+
     Raises:
         requests.RequestException: If conversion fails
     """
-    response = requests.post(YARRRML_URL, data={"yarrrml": mapping_data})
+    if isinstance(mapping_data, bytes):
+        yaml_str = mapping_data.decode('utf-8')
+    else:
+        yaml_str = mapping_data
+    yaml_str = normalize_yarrrml_nested_lists(yaml_str)
+    response = requests.post(YARRRML_URL, data={"yarrrml": yaml_str})
     response.raise_for_status()
     return response.text
 
@@ -1999,7 +2005,7 @@ async def yarrrmltorml(
     logging.info(f"POST /api/yarrrmltorml {final_mapping_url}")
     filedata, filename = open_file(final_mapping_url)
 
-    rules = requests.post(YARRRML_URL, data={"yarrrml": filedata}).text
+    rules = convert_yarrrml_to_rml(filedata)
     data_bytes = BytesIO(rules.encode())
     filename = filename.rsplit(".yaml", 1)[0] + "-rml.ttl"
     headers = {
@@ -2009,6 +2015,32 @@ async def yarrrmltorml(
     return TurtleResponse(content=data_bytes, headers=headers)
 
 
+@app.post("/api/normalizeyarrrml", response_class=PlainTextResponse, summary="Normalize YARRRML nested-list po: blocks to inline-list form", tags=["convert"])
+async def normalizeyarrrml(
+    file: Optional[UploadFile] = None,
+) -> PlainTextResponse:
+    """Accept a YARRRML file upload and return normalized YAML text.
+
+    Converts nested-list po: blocks (two-line form) to inline-list form
+    that yarrrml-parser accepts without silently dropping them.
+    """
+    if file is None:
+        raise HTTPException(status_code=422, detail="file upload is required")
+
+    raw = await file.read()
+    if len(raw) > 1_048_576:
+        raise HTTPException(status_code=413, detail="File exceeds 1 MB limit")
+
+    try:
+        yaml_str = raw.decode('utf-8')
+    except UnicodeDecodeError as exc:
+        raise HTTPException(status_code=422, detail="File must be UTF-8 encoded") from exc
+
+    normalized = normalize_yarrrml_nested_lists(yaml_str)
+    logging.info(f"POST /api/normalizeyarrrml filename={file.filename}")
+    return PlainTextResponse(content=normalized)
+
+
 @app.post("/api/createrdf", response_model=RDFResponse, summary="Create RDF from URL-accessible data", tags=["convert"])
 def create_rdf(
     req: Request,

test_createrdfupload.py

@@ -177,3 +177,115 @@ def test_createrdf_and_createrdfupload_produce_equivalent_graphs():
         f"  only in createrdf:       {po_url - po_upload}\n"
         f"  only in createrdfupload: {po_upload - po_url}"
     )
+
+
+# ---------------------------------------------------------------------------
+# YARRRML nested-list normalization integration tests
+# ---------------------------------------------------------------------------
+
+# Minimal YARRRML mapping using nested-list po: form (two-line form).
+# yarrrml-parser silently drops this form; normalization must convert it first.
+NESTED_LIST_MAPPING = """\
+prefixes:
+  ex: "http://example.org/"
+  rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+mappings:
+  Item:
+    sources:
+      - [source~csv]
+    s: ex:$(id)
+    po:
+      - - rdf:type
+        - ex:Item
+      - - ex:label
+        - $(label)
+"""
+
+INLINE_LIST_MAPPING = """\
+prefixes:
+  ex: "http://example.org/"
+  rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+mappings:
+  Item:
+    sources:
+      - [source~csv]
+    s: ex:$(id)
+    po:
+      - [rdf:type, ex:Item]
+      - [ex:label, $(label)]
+"""
+
+SIMPLE_CSV = "id,label\n1,hello\n"
+
+
+def test_normalizeyarrrml_endpoint_returns_200():
+    """POST /api/normalizeyarrrml returns 200 for valid YAML upload."""
+    response = httpx.post(
+        f"{BASE_URL}/api/normalizeyarrrml",
+        files={"file": ("mapping.yaml", NESTED_LIST_MAPPING, "text/plain")},
+    )
+    assert response.status_code == 200, response.text
+
+
+def test_normalizeyarrrml_converts_nested_to_inline():
+    """Endpoint converts nested-list po: form to inline-list form."""
+    response = httpx.post(
+        f"{BASE_URL}/api/normalizeyarrrml",
+        files={"file": ("mapping.yaml", NESTED_LIST_MAPPING, "text/plain")},
+    )
+    assert response.status_code == 200, response.text
+    assert response.text == INLINE_LIST_MAPPING
+
+
+def test_normalizeyarrrml_missing_file_returns_422():
+    """Endpoint rejects requests with no file upload."""
+    response = httpx.post(f"{BASE_URL}/api/normalizeyarrrml")
+    assert response.status_code == 422
+
+
+def test_createrdfupload_nested_list_mapping_produces_triples():
+    """Nested-list YARRRML mapping produces triples after normalization."""
+    response = post(
+        "/api/createrdfupload",
+        json={
+            "mapping_url": "data:text/plain;base64," + __import__("base64").b64encode(
+                NESTED_LIST_MAPPING.encode()
+            ).decode(),
+            "data_url": "http://example.org/source.csv",
+            "data_content": SIMPLE_CSV,
+        },
+    )
+    # 200 with triples means normalization fired and yarrrml-parser accepted the mapping
+    assert response.status_code == 200, response.text
+    body = response.json()
+    assert body["num_mappings_applied"] > 0, (
+        "Nested-list mapping produced zero mappings — normalization may not have fired"
+    )
+
+
+def test_createrdfupload_nested_and_inline_produce_equivalent_graphs():
+    """Nested-list and inline-list forms of the same mapping produce equal graphs."""
+    def run(mapping: str) -> Graph:
+        resp = post(
+            "/api/createrdfupload",
+            json={
+                "mapping_url": "data:text/plain;base64," + __import__("base64").b64encode(
+                    mapping.encode()
+                ).decode(),
+                "data_url": "http://example.org/source.csv",
+                "data_content": SIMPLE_CSV,
+            },
+        )
+        assert resp.status_code == 200, resp.text
+        return _strip_prov(Graph().parse(data=resp.json()["graph"], format="turtle"))
+
+    g_nested = run(NESTED_LIST_MAPPING)
+    g_inline = run(INLINE_LIST_MAPPING)
+
+    po_nested = {(str(p), str(o)) for _, p, o in g_nested}
+    po_inline = {(str(p), str(o)) for _, p, o in g_inline}
+    assert po_nested == po_inline, (
+        f"Nested vs inline graph mismatch:\n"
+        f"  only in nested: {po_nested - po_inline}\n"
+        f"  only in inline: {po_inline - po_nested}"
+    )

test_normalize_yarrrml.py

@@ -0,0 +1,133 @@
+"""Unit tests for normalize_yarrrml_nested_lists."""
+
+import glob
+import os
+import pytest
+from yarrrml_utils import normalize_yarrrml_nested_lists
+
+
+# ---------------------------------------------------------------------------
+# Happy-path: two-element nested list converted to inline list
+# ---------------------------------------------------------------------------
+
+NESTED_TWO = """\
+mappings:
+  Person:
+    s: ex:$(id)
+    po:
+      - - rdf:type
+        - ex:Person
+      - - foaf:name
+        - $(name)
+"""
+
+INLINE_TWO = """\
+mappings:
+  Person:
+    s: ex:$(id)
+    po:
+      - [rdf:type, ex:Person]
+      - [foaf:name, $(name)]
+"""
+
+
+def test_two_element_nested_converted():
+    assert normalize_yarrrml_nested_lists(NESTED_TWO) == INLINE_TWO
+
+
+# ---------------------------------------------------------------------------
+# Guard: three-element nested list must NOT be touched
+# ---------------------------------------------------------------------------
+
+NESTED_THREE = """\
+mappings:
+  Person:
+    po:
+      - - rdf:type
+        - ex:Person
+        - ex:Thing
+"""
+
+
+def test_three_element_nested_not_converted():
+    result = normalize_yarrrml_nested_lists(NESTED_THREE)
+    assert result == NESTED_THREE, (
+        "Three-element nested list must be left unchanged; got:\n" + result
+    )
+
+
+# ---------------------------------------------------------------------------
+# Already inline — must be idempotent
+# ---------------------------------------------------------------------------
+
+ALREADY_INLINE = """\
+mappings:
+  Person:
+    po:
+      - [rdf:type, ex:Person]
+"""
+
+
+def test_already_inline_unchanged():
+    assert normalize_yarrrml_nested_lists(ALREADY_INLINE) == ALREADY_INLINE
+
+
+# ---------------------------------------------------------------------------
+# Windows line endings normalized before regex
+# ---------------------------------------------------------------------------
+
+def test_crlf_normalized():
+    crlf = NESTED_TWO.replace('\n', '\r\n')
+    result = normalize_yarrrml_nested_lists(crlf)
+    assert result == INLINE_TWO
+
+
+# ---------------------------------------------------------------------------
+# Empty input
+# ---------------------------------------------------------------------------
+
+def test_empty_string():
+    assert normalize_yarrrml_nested_lists("") == ""
+
+
+# ---------------------------------------------------------------------------
+# Mixed: some nested, some already inline
+# ---------------------------------------------------------------------------
+
+MIXED = """\
+mappings:
+  A:
+    po:
+      - [rdf:type, ex:A]
+      - - foaf:name
+        - $(name)
+"""
+
+MIXED_NORMALIZED = """\
+mappings:
+  A:
+    po:
+      - [rdf:type, ex:A]
+      - [foaf:name, $(name)]
+"""
+
+
+def test_mixed_input():
+    assert normalize_yarrrml_nested_lists(MIXED) == MIXED_NORMALIZED
+
+
+# ---------------------------------------------------------------------------
+# Regression: examples/ YAML files contain no - - patterns to corrupt
+# ---------------------------------------------------------------------------
+
+EXAMPLES_DIR = os.path.join(os.path.dirname(__file__), "examples")
+
+
+@pytest.mark.parametrize("path", glob.glob(os.path.join(EXAMPLES_DIR, "*.yaml")))
+def test_examples_idempotent(path):
+    """Normalizing existing examples must not change them."""
+    with open(path, encoding="utf-8") as f:
+        content = f.read()
+    assert normalize_yarrrml_nested_lists(content) == content, (
+        f"{path} changed after normalization — check for unintended - - patterns"
+    )

yarrrml_utils.py

@@ -0,0 +1,25 @@
+"""Utilities for pre-processing YARRRML mapping files."""
+
+import re
+
+
+def normalize_yarrrml_nested_lists(yaml_str: str) -> str:
+    """Convert YARRRML nested-list po: blocks to inline-list form.
+
+    yarrrml-parser silently drops the two-line nested-list form:
+        - - predicate
+          - object
+    This function converts it to the inline-list form that is accepted:
+        - [predicate, object]
+
+    Three-element (or longer) nested lists are left unchanged — the
+    negative lookahead ensures only exactly two-element pairs are touched.
+    Windows line endings are normalised before the regex runs.
+    """
+    yaml_str = yaml_str.replace('\r\n', '\n')
+    return re.sub(
+        r'^( +)- - (.+)\n\1  - (.+)$(?!\n\1  -)',
+        lambda m: f"{m.group(1)}- [{m.group(2)}, {m.group(3)}]",
+        yaml_str,
+        flags=re.MULTILINE,
+    )