EXTENDS validation with feasibility-pruned partition search

.github/workflows/main.yaml

@@ -4,7 +4,7 @@ env:
 on:
   push:
     branches:
-      - main
+      - master
   pull_request:
   workflow_dispatch:
 
@@ -47,6 +47,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
+          submodules: true
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
@@ -66,6 +67,8 @@ jobs:
       - name: Generate coverage results
         # Set bash shell to fail correctly on Windows https://github.com/actions/runner-images/issues/6668
         shell: bash
+        env:
+          SKIP_EXTERNAL_URLS: "true"
         run: |
           uv run coverage run -m pytest
           uv run coverage xml

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "tests/data/shexTest"]
 	path = tests/data/shexTest
-	url = git@github.com:shexSpec/shexTest.git
+	url = https://github.com/shexSpec/shexTest.git

ancilliary/earlreport.py

@@ -54,13 +54,19 @@ def __init__(self, author: URIRef):
         self.g = Graph()
         self.g.parse(data=header, format="turtle")
         self.author = author
+        # one timestamp for the whole report and sequential bnode labels: the
+        # turtle serializer orders bnode stanzas by label, so this keeps the
+        # serialized assertions in manifest order instead of shuffling per run
+        self.issued = datetime.datetime.utcnow().isoformat()
+        self._sequence = 0
 
     def add(self, s: Node, p: URIRef, o: Node) -> "EARLPage":
         self.g.add((s, p, o))
         return self
 
     def add_test_result(self, test_entry: str, status: str) -> None:
-        entry = BNode()
+        entry = BNode(f"assertion{self._sequence:05d}")
+        self._sequence += 1
         self.add(entry, RDF.type, EARL.Assertion)\
             .add(entry, EARL.assertedBy, self.author)\
             .add(entry, EARL.test, MFST[test_entry])\
@@ -72,7 +78,7 @@ def _add_result(self, entry: BNode, status: bool) -> None:
         rslt = BNode()
         self.add(rslt, RDF.type, EARL.TestResult)\
             .add(rslt, EARL.outcome, EARL[status])\
-            .add(rslt, DC.date, Literal(datetime.datetime.utcnow().isoformat()))\
+            .add(rslt, DC.date, Literal(self.issued))\
             .add(entry, EARL.result, rslt)
 
     def __str__(self) -> str:

pyproject.toml

@@ -64,6 +64,12 @@ dev = [
     "coverage",
 ]
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+# tests/data holds test *data* (including the shexTest submodule, which ships
+# its own pytest suite) — never collect from it
+addopts = "--ignore=tests/data"
+
 [tool.black]
 line-length = 120
 target-version = ["py310", "py311", "py312", "py313",  "py314"]
@@ -96,4 +102,6 @@ deps = [
 ]
 commands = [
   ["codespell", "{posargs}"]
-]
+]
+[tool.uv.sources]
+pyshexc = { git = "https://github.com/ericprud/grammar-python-antlr-linkml.git", branch = "extends-shaperef" }

pyshex/__init__.py

@@ -1,3 +1,15 @@
+import rdflib
+
+# A validator must see literals as written: rdflib's default lexical normalization
+# rewrites e.g. "TRUE"^^xsd:boolean to "true" and "1E0"^^xsd:decimal to "1" at parse
+# time, which would hide invalid lexical forms from datatype validation.
+rdflib.NORMALIZE_LITERALS = False
+
+# ... and the N3/Turtle tokenizer separately erases the lexical form of bare numerics
+# (00 -> "0"^^xsd:integer)
+from pyshex.utils.rdflib_lexical_fidelity import install as _install_lexical_fidelity
+_install_lexical_fidelity()
+
 from pyshex.prefixlib import PrefixLibrary, standard_prefixes, known_prefixes
 from pyshex.shex_evaluator import ShExEvaluator
 

pyshex/prefixlib.py

@@ -66,14 +66,23 @@ def add_shex(self, schema: str) -> "PrefixLibrary":
 
     def add_rdf(self, rdf: str | Graph, format: str | None = "turtle") -> "PrefixLibrary":
         if not isinstance(rdf, Graph):
-            g = Graph()
+            # "core" keeps the pre-rdflib-6 binding set (rdf, rdfs, xsd, ...) rather
+            # than the ~30 namespaces rdflib now binds by default.  The json-ld
+            # parser re-binds the full default set mid-parse, so for that format
+            # anything beyond core that matches a default binding is dropped —
+            # at the cost of losing @context prefixes identical to a default.
+            g = Graph(bind_namespaces="core")
+            injectable = set(Graph().namespace_manager.namespaces()) - \
+                set(g.namespace_manager.namespaces()) if format and 'json-ld' in format else set()
             if '\n' in rdf or '\r' in rdf or ' ' in rdf:
                 g.parse(data=rdf, format=format)
             else:
                 g.parse(rdf, format=format)
+            namespaces = [(k, v) for k, v in g.namespace_manager.namespaces()
+                          if (k, v) not in injectable]
         else:
-            g = rdf
-        for k, v in g.namespace_manager.namespaces():
+            namespaces = list(rdf.namespace_manager.namespaces())
+        for k, v in namespaces:
             setattr(self, k.upper(), Namespace(v))
         return self
 

pyshex/shape_expressions_language/p5_2_validation_definition.py

@@ -3,6 +3,7 @@
 
 from pyshex.parse_tree.parse_node import ParseNode
 from pyshex.shape_expressions_language.p5_3_shape_expressions import satisfies
+from pyshex.shape_expressions_language.p5_7_semantic_actions import semActsSatisfied
 from pyshex.shape_expressions_language.p5_context import Context
 from pyshex.shapemap_structure_and_language.p1_notation_and_terminology import Node
 from pyshex.shapemap_structure_and_language.p3_shapemap_structure import FixedShapeMap, START, nodeSelector
@@ -21,6 +22,8 @@ def isValid(cntxt: Context, m: FixedShapeMap) -> tuple[bool, list[str]]:
     """
     if not cntxt.is_valid:
         return False, cntxt.error_list
+    if not semActsSatisfied(getattr(cntxt.schema, 'startActs', None), cntxt):
+        return False, ["Schema startActs semantic action failed"]
     parse_nodes = []
     for nodeshapepair in m:
         n = nodeshapepair.nodeSelector

pyshex/shape_expressions_language/p5_3_shape_expressions.py

@@ -4,7 +4,8 @@
 from pyjsg.jsglib import isinstance_
 
 from pyshex.shape_expressions_language.p5_4_node_constraints import satisfiesNodeConstraint
-from pyshex.shape_expressions_language.p5_5_shapes_and_triple_expressions import satisfiesShape
+from pyshex.shape_expressions_language.p5_5_shapes_and_triple_expressions import satisfiesShape, \
+    _active_restriction, suspended_inherited_closed
 from pyshex.shape_expressions_language.p5_context import Context, DebugContext
 from pyshex.shapemap_structure_and_language.p1_notation_and_terminology import Node
 from pyshex.utils.trace_utils import trace_satisfies
@@ -33,7 +34,15 @@ def satisfies(cntxt: Context, n: Node, se: ShExJ.shapeExpr) -> bool:
           .. note:: Where is the documentation on recursion?  All I can find is
            `5.9.4 Recursion Example <http://shex.io/shex-semantics/#example-recursion>`_
           """
-    if isinstance(se, ShExJ.NodeConstraint):
+    if (isinstance(se, ShExJ.ShapeDecl) or getattr(se, 'abstract', None) or getattr(se, 'restricts', None)
+        or (getattr(se, 'id', None) is not None and _extensions_index(cntxt).get(str(se.id)))) \
+            and id(se) not in getattr(cntxt, '_decl_bypass', set()):
+        # ShExJ 2.1 puts abstract/restricts on ShapeDecl; older parsers put them
+        # directly on the shape expression.  Both flavours are handled here.
+        rval = satisfiesShapeDecl(cntxt, n, se)
+    elif isinstance(se, ShExJ.ShapeDecl):
+        rval = satisfies(cntxt, n, se.shapeExpr)
+    elif isinstance(se, ShExJ.NodeConstraint):
         rval = satisfiesNodeConstraint(cntxt, n, se)
     elif isinstance(se, ShExJ.Shape):
         rval = satisfiesShape(cntxt, n, se)
@@ -52,6 +61,89 @@ def satisfies(cntxt: Context, n: Node, se: ShExJ.shapeExpr) -> bool:
     return rval
 
 
+def satisfiesShapeDecl(cntxt: Context, n: Node, se) -> bool:
+    """A ShapeDecl's restricts are additional conjuncts: the node must satisfy the
+    declared shape expression and every restricted shape (cf. shex.js validateShapeDecl,
+    which conjoins restricts + shapeExpr as a ShapeAnd).  An abstract declaration cannot
+    be satisfied directly: some non-abstract declaration extending it must be satisfied
+    instead -- except when the declaration is being evaluated as an EXTENDS target
+    against its allocated part of the neighbourhood, where the extension mechanism has
+    already chosen it (cf. shex.js validateDescendants under a subGraph)."""
+
+    inner = se.shapeExpr if isinstance(se, ShExJ.ShapeDecl) else se
+
+    def direct() -> bool:
+        # ShExJ 2.1 moved labels onto ShapeDecl: a ShapeExternal no longer carries the
+        # id that external resolution is keyed on, so pass the declaration's down.
+        if isinstance(inner, ShExJ.ShapeExternal) and getattr(inner, 'id', None) is None:
+            inner.id = se.id
+        if inner is se:
+            # legacy flavour: the abstract/restricts live on the expression itself;
+            # bypass the declaration branch when recursing into it
+            if not hasattr(cntxt, '_decl_bypass'):
+                cntxt._decl_bypass = set()
+            cntxt._decl_bypass.add(id(se))
+            try:
+                rslt = satisfies(cntxt, n, inner)
+            finally:
+                cntxt._decl_bypass.discard(id(se))
+        else:
+            rslt = satisfies(cntxt, n, inner)
+        return rslt and all(satisfies(cntxt, n, r) for r in (getattr(se, 'restricts', None) or []))
+
+    if _active_restriction(cntxt, n) is not None:
+        return direct()
+    if (not getattr(se, 'abstract', None)) and direct():
+        return True
+    return any(satisfiesShapeDecl(cntxt, n, d) for d in _non_abstract_descendants(cntxt, se))
+
+
+def _extensions_index(cntxt: Context) -> dict[str, list]:
+    """parent shape label -> declarations that extend it (directly), computed once."""
+    if not hasattr(cntxt, '_extensions_children'):
+        children: dict[str, list] = {}
+        for decl in cntxt.schema.shapes or []:
+            if not isinstance(decl, ShExJ.ShapeDecl) and getattr(decl, 'id', None) is None:
+                continue
+            parents: set[str] = set()
+
+            def walk(e) -> None:
+                if e is None or isinstance_(e, ShExJ.shapeExprLabel):
+                    return
+                if isinstance(e, (ShExJ.ShapeAnd, ShExJ.ShapeOr)):
+                    for nested in e.shapeExprs:
+                        walk(nested)
+                elif isinstance(e, ShExJ.ShapeNot):
+                    walk(e.shapeExpr)
+                elif isinstance(e, ShExJ.Shape):
+                    for parent in getattr(e, 'extends', None) or []:
+                        parents.add(str(parent))
+
+            walk(decl.shapeExpr if isinstance(decl, ShExJ.ShapeDecl) else decl)
+            for parent in parents:
+                children.setdefault(parent, []).append(decl)
+        cntxt._extensions_children = children
+    return cntxt._extensions_children
+
+
+def _non_abstract_descendants(cntxt: Context, decl: ShExJ.ShapeDecl) -> list:
+    """The non-abstract declarations that transitively extend decl."""
+    index = _extensions_index(cntxt)
+    result: list = []
+    seen: set[str] = {str(decl.id)}
+    stack: list[str] = [str(decl.id)]
+    while stack:
+        for child in index.get(stack.pop(), []):
+            child_id = str(child.id)
+            if child_id in seen:
+                continue
+            seen.add(child_id)
+            stack.append(child_id)
+            if not getattr(child, 'abstract', None):
+                result.append(child)
+    return result
+
+
 @trace_satisfies()
 def notSatisfies(cntxt: Context, n: Node, se: ShExJ.shapeExpr, _: DebugContext) -> bool:
     return not satisfies(cntxt, n, se)
@@ -66,13 +158,15 @@ def satisifesShapeOr(cntxt: Context, n: Node, se: ShExJ.ShapeOr, _: DebugContext
 @trace_satisfies()
 def satisfiesShapeAnd(cntxt: Context, n: Node, se: ShExJ.ShapeAnd, _: DebugContext) -> bool:
     """ Se is a ShapeAnd and for every shape expression se2 in shapeExprs, satisfies(n, se2, G, m) """
-    return all(satisfies(cntxt, n, se2) for se2 in se.shapeExprs)
+    with suspended_inherited_closed(cntxt, n):
+        return all(satisfies(cntxt, n, se2) for se2 in se.shapeExprs)
 
 
 @trace_satisfies()
 def satisfiesShapeNot(cntxt: Context, n: Node, se: ShExJ.ShapeNot, _: DebugContext) -> bool:
     """ Se is a ShapeNot and for the shape expression se2 at shapeExpr, notSatisfies(n, se2, G, m) """
-    return not satisfies(cntxt, n, se.shapeExpr)
+    with suspended_inherited_closed(cntxt, n):
+        return not satisfies(cntxt, n, se.shapeExpr)
 
 
 @trace_satisfies(True)

pyshex/shape_expressions_language/p5_4_node_constraints.py

@@ -10,7 +10,8 @@
 from pyshex.shape_expressions_language.p5_context import Context, DebugContext
 from pyshex.shapemap_structure_and_language.p1_notation_and_terminology import Node
 from pyshex.sparql11_query.p17_1_operand_data_types import is_sparql_operand_datatype, is_numeric
-from pyshex.utils.datatype_utils import can_cast_to, total_digits, fraction_digits, pattern_match, map_object_literal
+from pyshex.utils.datatype_utils import can_cast_to, total_digits, fraction_digits, is_valid_lexical_form, \
+    pattern_match, map_object_literal
 from pyshex.utils.trace_utils import trace_satisfies
 from pyshex.utils.value_set_utils import objectValueMatches, uriref_startswith_iriref, uriref_matches_iriref
 
@@ -70,8 +71,16 @@ def nodeSatisfiesDataType(cntxt: Context, n: Node, nc: ShExJ.NodeConstraint, c:
         cntxt.dump_bnode(n)
         return False
     actual_datatype = _datatype(n)
-    if actual_datatype == str(nc.datatype) or \
-        (is_sparql_operand_datatype(nc.datatype) and can_cast_to(n, nc.datatype)):
+    if actual_datatype == str(nc.datatype):
+        # "an XML schema string with a value of the lexical form of n can be cast to the
+        # target type" -- a matching datatype IRI is not enough if the lexical form is
+        # outside the datatype's lexical space (rdflib leaves such literals ill-typed
+        # but does not validate value ranges, so check both here)
+        if is_valid_lexical_form(n) is False:
+            cntxt.fail_reason = f"Invalid lexical form for {nc.datatype}: '{n}'"
+            return False
+        return True
+    if is_sparql_operand_datatype(nc.datatype) and can_cast_to(n, nc.datatype):
         return True
     cntxt.fail_reason = f"Datatype mismatch - expected: {nc.datatype} actual: {actual_datatype}"
     return False
@@ -256,18 +265,40 @@ def _nodeSatisfiesValue(cntxt: Context, n: Node, vsv: ShExJ.valueSetValue) -> bo
 
     if isinstance(vsv, ShExJ.LiteralStemRange):
         exclusions = vsv.exclusions if vsv.exclusions is not None else []
-        return nodeInLiteralStem(cntxt, n, vsv.stem) and not any(str(n) == excl for excl in exclusions)
+        return nodeInLiteralStem(cntxt, n, vsv.stem) and not any(
+            (isinstance(n, Literal) and str(n).startswith(str(excl.stem)))
+            if isinstance(excl, ShExJ.LiteralStem) else str(n) == str(excl)
+            for excl in exclusions)
 
     if isinstance(vsv, ShExJ.LanguageStem):
         return nodeInLanguageStem(cntxt, n, vsv.stem)
 
     if isinstance(vsv, ShExJ.LanguageStemRange):
         exclusions = vsv.exclusions if vsv.exclusions is not None else []
-        return nodeInLanguageStem(cntxt, n, vsv.stem) and not any(str(n) == str(excl) for excl in exclusions)
+        return nodeInLanguageStem(cntxt, n, vsv.stem) and not any(
+            _language_tag_matches_stem(_language_tag(n), str(excl.stem))
+            if isinstance(excl, ShExJ.LanguageStem) else
+            (_language_tag(n) is not None and _language_tag(n).lower() == str(excl).lower())
+            for excl in exclusions)
 
     return False
 
 
+def _language_tag(n: Node) -> str | None:
+    return n.language if isinstance(n, Literal) else None
+
+
+def _language_tag_matches_stem(tag: str | None, stem: str) -> bool:
+    """`RFC 4647 basic filtering <https://datatracker.ietf.org/doc/html/rfc4647#section-3.3.1>`_:
+    a language range matches a tag it equals (case-insensitively) or prefixes at a
+    subtag ('-') boundary; the empty range matches any language-tagged string."""
+    if tag is None:
+        return False
+    if stem == '':
+        return True
+    return tag.lower() == stem.lower() or tag.lower().startswith(stem.lower() + '-')
+
+
 def nodeInIriStem(_: Context, n: Node, s: ShExJ.IriStem) -> bool:
     """
        **nodeIn**: asserts that an RDF node n is equal to an RDF term s or is in a set defined by a
@@ -292,7 +323,7 @@ def nodeInLiteralStem(_: Context, n: Node, s: ShExJ.LiteralStem) -> bool:
          #) `n` is an :py:class:`rdflib.Literal` and fn:starts-with(`n`, `s`)
      """
     return isinstance(s, ShExJ.Wildcard) or \
-        (isinstance(n, Literal) and str(n.value).startswith(str(s)))
+        (isinstance(n, Literal) and str(n).startswith(str(s)))
 
 
 def nodeInLanguageStem(_: Context, n: Node, s: ShExJ.LanguageStem) -> bool:
@@ -303,10 +334,11 @@ def nodeInLanguageStem(_: Context, n: Node, s: ShExJ.LanguageStem) -> bool:
 
         The expression `nodeInLanguageStem(n, s)` is satisfied iff:
          #) `s` is a :py:class:`ShExJ.WildCard` or
-         #) `n` is a language-tagged string and fn:starts-with(`n.language`, `s`)
+         #) `n` is a language-tagged string whose tag matches `s` as a basic language
+            range (equality or a '-' subtag boundary -- "fr" matches "fr-BE" but not
+            "frc")
     """
-    return isinstance(s, ShExJ.Wildcard) or \
-        (isinstance(n, Literal) and n.language is not None and str(n.language).startswith(str(s)))
+    return isinstance(s, ShExJ.Wildcard) or _language_tag_matches_stem(_language_tag(n), str(s))
 
 
 def nodeInBnodeStem(_cntxt: Context, _n: Node, _s: str | ShExJ.Wildcard) -> bool: