From cffee395f6f1ec88b6b0359142ab543c57ace24b Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 10 Mar 2026 11:14:01 -0700 Subject: [PATCH 01/90] build(hatch-env): specify env for conversion to LinkML and back Specify Hatch-managed env for auto converting `dandischema.models` to LinkML schema and back to Pydantic models --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index dca8b6b2..1d534fcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,12 @@ all = [ installer = "uv" python = "3.10" +# Env for auto converting `dandischema.models` to LinkML schema and back to Pydantic models +[tool.hatch.envs.linkml-auto-converted] +dependencies = [ + "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" +] + [tool.setuptools.packages.find] namespaces = true include = ["dandischema*"] From b5644c1a3558923707e5c2d4c08ab861876a6081 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 10 Mar 2026 11:43:36 -0700 Subject: [PATCH 02/90] build(hatch-env): provide script to translate `dandischema.models` Provide script to translate `dandischema.models` in to a LinkML schema and overly it with definition provided by an overlay file. --- dandischema/models_overlay.yaml | 4 ++++ pyproject.toml | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 dandischema/models_overlay.yaml diff --git a/dandischema/models_overlay.yaml b/dandischema/models_overlay.yaml new file mode 100644 index 00000000..69b1c3e2 --- /dev/null +++ b/dandischema/models_overlay.yaml @@ -0,0 +1,4 @@ +name: dandi-schema +id: https://schema.dandiarchive.org/s/dandi/v0.7 +version: 0.7.0 +status: eunal:concept-status/DRAFT diff --git a/pyproject.toml b/pyproject.toml index 1d534fcc..962caf07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,8 @@ python = "3.10" dependencies = [ "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" ] +[tool.hatch.envs.linkml-auto-converted.scripts] +2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml -o dandischema/models.yaml dandischema.models" [tool.setuptools.packages.find] namespaces = true From cfe3beacadeb5923a32b82a7187f724c268390f5 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 10 Mar 2026 12:22:13 -0700 Subject: [PATCH 03/90] build(hatch-env): provide script to translate `dandischema/models.yaml` Provide script to translate `dandischema/models.yaml` back to Pydantic models and store them in `dandischema/models.py` --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 962caf07..d51e4a19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,10 +65,12 @@ python = "3.10" # Env for auto converting `dandischema.models` to LinkML schema and back to Pydantic models [tool.hatch.envs.linkml-auto-converted] dependencies = [ + "black", # This allows `gen-pydantic` to format the generated Pydantic models with Black formatting "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" ] [tool.hatch.envs.linkml-auto-converted.scripts] 2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml -o dandischema/models.yaml dandischema.models" +2pydantic = "gen-pydantic --black dandischema/models.yaml > dandischema/models.py" [tool.setuptools.packages.find] namespaces = true From 73c46b6a4690ce492282b407c44d8052d69082e7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 13 Mar 2026 16:42:59 -0400 Subject: [PATCH 04/90] Now we will sanitize the converted model addresses --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d51e4a19..5d6dcd32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ dependencies = [ "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" ] [tool.hatch.envs.linkml-auto-converted.scripts] -2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml -o dandischema/models.yaml dandischema.models" +2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' > dandischema/models.yaml " 2pydantic = "gen-pydantic --black dandischema/models.yaml > dandischema/models.py" [tool.setuptools.packages.find] From c79b1e2694c3d86b0813f84fca4455c37b9e58e0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 13 Mar 2026 18:32:09 -0400 Subject: [PATCH 05/90] more sanitization --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5d6dcd32..1792f36e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ dependencies = [ "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" ] [tool.hatch.envs.linkml-auto-converted.scripts] -2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' > dandischema/models.yaml " +2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -e 's,\\(.\\+\\):[0-9]\\+,\\1:NUMBER,g' > dandischema/models.yaml" 2pydantic = "gen-pydantic --black dandischema/models.yaml > dandischema/models.py" [tool.setuptools.packages.find] From 0b2b51dff340539dfea196319bf8c526d17ec71b Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 12 Mar 2026 20:07:21 -0700 Subject: [PATCH 06/90] feat: provide `prefixes` definition in the overlay file These prefixes are copied from https://github.com/dandi/schema/blob/master/releases/0.7.0/context.json --- dandischema/models_overlay.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dandischema/models_overlay.yaml b/dandischema/models_overlay.yaml index 69b1c3e2..f3e477bd 100644 --- a/dandischema/models_overlay.yaml +++ b/dandischema/models_overlay.yaml @@ -2,3 +2,27 @@ name: dandi-schema id: https://schema.dandiarchive.org/s/dandi/v0.7 version: 0.7.0 status: eunal:concept-status/DRAFT + +prefixes: + dandi: http://schema.dandiarchive.org/ + dcite: http://schema.dandiarchive.org/datacite/ + dandiasset: http://dandiarchive.org/asset/ + DANDI: http://dandiarchive.org/dandiset/ + dct: http://purl.org/dc/terms/ + owl: http://www.w3.org/2002/07/owl# + rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# + rdfa: http://www.w3.org/ns/rdfa# + rdfs: http://www.w3.org/2000/01/rdf-schema# + schema: http://schema.org/ + xsd: http://www.w3.org/2001/XMLSchema# + skos: http://www.w3.org/2004/02/skos/core# + prov: http://www.w3.org/ns/prov# + pav: http://purl.org/pav/ + nidm: http://purl.org/nidash/nidm# + uuid: http://uuid.repronim.org/ + rs: http://schema.repronim.org/ + RRID: "https://scicrunch.org/resolver/RRID:" + ORCID: https://orcid.org/ + ROR: https://ror.org/ + PATO: http://purl.obolibrary.org/obo/PATO_ + spdx: http://spdx.org/licenses/ From 6cb2f826c6c011d72ce7f7d9d91a985417da19ff Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 13 Mar 2026 17:05:48 -0400 Subject: [PATCH 07/90] Added linkml: prefix and sorted all the entries --- dandischema/models_overlay.yaml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/dandischema/models_overlay.yaml b/dandischema/models_overlay.yaml index f3e477bd..d10006bc 100644 --- a/dandischema/models_overlay.yaml +++ b/dandischema/models_overlay.yaml @@ -4,25 +4,26 @@ version: 0.7.0 status: eunal:concept-status/DRAFT prefixes: - dandi: http://schema.dandiarchive.org/ - dcite: http://schema.dandiarchive.org/datacite/ dandiasset: http://dandiarchive.org/asset/ DANDI: http://dandiarchive.org/dandiset/ + dandi: http://schema.dandiarchive.org/ + dcite: http://schema.dandiarchive.org/datacite/ dct: http://purl.org/dc/terms/ + linkml: https://w3id.org/linkml/ + nidm: http://purl.org/nidash/nidm# + ORCID: https://orcid.org/ owl: http://www.w3.org/2002/07/owl# - rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# + PATO: http://purl.obolibrary.org/obo/PATO_ + pav: http://purl.org/pav/ + prov: http://www.w3.org/ns/prov# rdfa: http://www.w3.org/ns/rdfa# + rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# rdfs: http://www.w3.org/2000/01/rdf-schema# + ROR: https://ror.org/ + RRID: "https://scicrunch.org/resolver/RRID:" + rs: http://schema.repronim.org/ schema: http://schema.org/ - xsd: http://www.w3.org/2001/XMLSchema# skos: http://www.w3.org/2004/02/skos/core# - prov: http://www.w3.org/ns/prov# - pav: http://purl.org/pav/ - nidm: http://purl.org/nidash/nidm# - uuid: http://uuid.repronim.org/ - rs: http://schema.repronim.org/ - RRID: "https://scicrunch.org/resolver/RRID:" - ORCID: https://orcid.org/ - ROR: https://ror.org/ - PATO: http://purl.obolibrary.org/obo/PATO_ spdx: http://spdx.org/licenses/ + uuid: http://uuid.repronim.org/ + xsd: http://www.w3.org/2001/XMLSchema# From 76526a9d49c2d591b57d1927bb7f6b271e318888 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 13 Mar 2026 17:08:12 -0400 Subject: [PATCH 08/90] Provide default prefix to be custom dandi_default --- dandischema/models_overlay.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dandischema/models_overlay.yaml b/dandischema/models_overlay.yaml index d10006bc..983cbaaf 100644 --- a/dandischema/models_overlay.yaml +++ b/dandischema/models_overlay.yaml @@ -27,3 +27,5 @@ prefixes: spdx: http://spdx.org/licenses/ uuid: http://uuid.repronim.org/ xsd: http://www.w3.org/2001/XMLSchema# + +default_prefix: dandi_default From 7fa95c24c88d614c156a91d1ef2b955b500b7241 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 13 Mar 2026 18:18:09 -0400 Subject: [PATCH 09/90] use portable ERE sed pattern for normalizing memory addresses The previous BRE pattern used `\+` (GNU sed extension) which silently fails on macOS BSD sed. Switch to `-E` (extended regex) with POSIX character class `[^[:space:]]` instead of `\S` (also unsupported by BSD sed), making the normalization work on both macOS and Linux. Co-Authored-By: Claude Sonnet 4.6 Expand comment for linkml-auto-converted hatch env with usage instructions Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1792f36e..316c271e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,14 +63,17 @@ installer = "uv" python = "3.10" # Env for auto converting `dandischema.models` to LinkML schema and back to Pydantic models +# To invoke conversion scripts provided by the environment, execute the following respective commands: +# `hatch run linkml-auto-converted:2linkml` # Convert dandischema.models -> dandischema/models.yaml +# `hatch run linkml-auto-converted:2pydantic` # Convert dandischema/models.yaml -> dandischema/models_linkml.py [tool.hatch.envs.linkml-auto-converted] dependencies = [ "black", # This allows `gen-pydantic` to format the generated Pydantic models with Black formatting "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" ] [tool.hatch.envs.linkml-auto-converted.scripts] -2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -e 's,\\(.\\+\\):[0-9]\\+,\\1:NUMBER,g' > dandischema/models.yaml" -2pydantic = "gen-pydantic --black dandischema/models.yaml > dandischema/models.py" +2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' > dandischema/models.yaml" +2pydantic = "gen-pydantic --black dandischema/models.yaml > dandischema/models_linkml.py" [tool.setuptools.packages.find] namespaces = true From fad68793ea7408dba05467fb74b7e12996206b7c Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 16 Mar 2026 16:57:13 -0700 Subject: [PATCH 10/90] fix: correct `default_prefix` to `dandi` There is no prefix defined as `dandi_default`. The intended default prefix is `dandi` --- dandischema/models_overlay.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandischema/models_overlay.yaml b/dandischema/models_overlay.yaml index 983cbaaf..6bcb0144 100644 --- a/dandischema/models_overlay.yaml +++ b/dandischema/models_overlay.yaml @@ -28,4 +28,4 @@ prefixes: uuid: http://uuid.repronim.org/ xsd: http://www.w3.org/2001/XMLSchema# -default_prefix: dandi_default +default_prefix: dandi From 5f78b87873728bccddc2c4ea469469d1c9003095 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 13 Mar 2026 18:22:41 -0400 Subject: [PATCH 11/90] Adding models_importstab.py so we could use it to import from converted and some symbols from _orig for now we do it so it does not overlay models.py since then git is unable to track renames --- dandischema/models_importstab.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 dandischema/models_importstab.py diff --git a/dandischema/models_importstab.py b/dandischema/models_importstab.py new file mode 100644 index 00000000..6815ca49 --- /dev/null +++ b/dandischema/models_importstab.py @@ -0,0 +1,9 @@ +from .models_linkml import * # noqa: F401,F403 +from .models_orig import DANDI_INSTANCE_URL_PATTERN # noqa: F401 + +# TODO: temporary imports of consts etc which might need to be 'redone' +# so we do not duplicate them + + +# TODO: do the extra tune ups like linking extra validations etc, +# potentially copied from models_orig.py From 0208c9c6eed6c6e4431f33e5d6dfb7b6fba3bffe Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 17:13:26 -0400 Subject: [PATCH 12/90] Adding the tool to assist in establishing "merges" with regeneration into linkml-auto-converted --- tools/linkml_conversion | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 tools/linkml_conversion diff --git a/tools/linkml_conversion b/tools/linkml_conversion new file mode 100755 index 00000000..c34e803c --- /dev/null +++ b/tools/linkml_conversion @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Script which will force us to apply the changes we have +# + +set -eu + +ver=$(git describe linkml-conversion) + +# ensure we are clean to no side effects and are in +test -z "$(git status --porcelain)" || { echo "Git tree is dirty, aborting."; exit 1; } + +git checkout linkml-auto-converted + +git merge -s ours --no-commit linkml-conversion + +git read-tree -m -u linkml-conversion + +hatch run linkml-auto-converted:2linkml +hatch run linkml-auto-converted:2pydantic + +# add (re)generated files +git add dandischema/models_linkml.py dandischema/models.yaml + +# add our stab +git mv dandischema/models_importstab.py dandischema/models.py + +git commit -m '"Merged" with reconversion linkml-conversion $ver' -a From 488e2e94f74b2101e36c0ab74ed4c79d234e1721 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 17:18:02 -0400 Subject: [PATCH 13/90] We are doomed to use --no-verify for now as converted python code has flake8 issues --- tools/linkml_conversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index c34e803c..282d0565 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -25,4 +25,4 @@ git add dandischema/models_linkml.py dandischema/models.yaml # add our stab git mv dandischema/models_importstab.py dandischema/models.py -git commit -m '"Merged" with reconversion linkml-conversion $ver' -a +git commit -m '"Merged" with reconversion linkml-conversion $ver' -a --no-verify From 0ba5ad757e265992da110424ab3c2b83b3398bb7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 17:22:05 -0400 Subject: [PATCH 14/90] BF the commit version specification --- tools/linkml_conversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index 282d0565..21e74f09 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -25,4 +25,4 @@ git add dandischema/models_linkml.py dandischema/models.yaml # add our stab git mv dandischema/models_importstab.py dandischema/models.py -git commit -m '"Merged" with reconversion linkml-conversion $ver' -a --no-verify +git commit -m "'Merged' with reconversion linkml-conversion $ver" -a --no-verify From 2faf932ae562cd09ea0544810c9454812e58e3e1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 17:38:34 -0400 Subject: [PATCH 15/90] Implement poor man patch queue --- tools/linkml_conversion | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index 21e74f09..abdb9f26 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -16,6 +16,19 @@ git merge -s ours --no-commit linkml-conversion git read-tree -m -u linkml-conversion +# Poor man patch queue implementation +# Edit this list if you want to merge or drop PRs branches to be patched with. +# Order matters +branches_to_merge=( remove-discriminated-unions ) + +# determine remote name based on what remote linkml-conversion is +# tracked from which should correspond to our main repo! +remote=$(git config branch.linkml-conversion.remote) + +for b in $"{branches_to_merge[@]}"; do + git diff "$remote"/master...$b | patch -p1 +done + hatch run linkml-auto-converted:2linkml hatch run linkml-auto-converted:2pydantic @@ -25,4 +38,7 @@ git add dandischema/models_linkml.py dandischema/models.yaml # add our stab git mv dandischema/models_importstab.py dandischema/models.py -git commit -m "'Merged' with reconversion linkml-conversion $ver" -a --no-verify +git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches + +branches merged: ${branches_to_merge[@]} +" -a --no-verify From c177479760b4ebb4d555cb427473dfcb708adfa5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 17:49:35 -0400 Subject: [PATCH 16/90] reflect that I rewritten how we handle patching we had to maintain original filename for models.py to apply patches easily --- tools/linkml_conversion | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index abdb9f26..c600b1df 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -35,7 +35,8 @@ hatch run linkml-auto-converted:2pydantic # add (re)generated files git add dandischema/models_linkml.py dandischema/models.yaml -# add our stab +# add our stab and rename original models.py +git mv dandischema/models.py dandischema/models_orig.py git mv dandischema/models_importstab.py dandischema/models.py git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches From 54cf276ede8a9da021395715e7d0935e101fc722 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 17:52:23 -0400 Subject: [PATCH 17/90] Inform on what we are doing --- tools/linkml_conversion | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index c600b1df..bd80abd0 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -25,8 +25,12 @@ branches_to_merge=( remove-discriminated-unions ) # tracked from which should correspond to our main repo! remote=$(git config branch.linkml-conversion.remote) -for b in $"{branches_to_merge[@]}"; do - git diff "$remote"/master...$b | patch -p1 +for b in ${branches_to_merge[@]}; do + b_ver=$(git describe "$remote"/$b) + echo "Applying branch $b with following differences" + git diff --stat "$remote"/master..."$remote"/$b + + git diff "$remote"/master..."$remote"/$b | patch -p1 done hatch run linkml-auto-converted:2linkml From 5eb4a91f695af0b2dbb948335e4e8bc9671d5f13 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 18:08:21 -0400 Subject: [PATCH 18/90] Now do patch diff of master in as well for good measure --- tools/linkml_conversion | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index bd80abd0..f2b8175b 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -5,11 +5,21 @@ set -eu -ver=$(git describe linkml-conversion) - # ensure we are clean to no side effects and are in test -z "$(git status --porcelain)" || { echo "Git tree is dirty, aborting."; exit 1; } +# determine remote name based on what remote linkml-conversion is +# tracked from which should correspond to our main repo! +remote=$(git config branch.linkml-conversion.remote) + +git checkout linkml-conversion +# ensure up to date! +git pull --ff-only + +ver=$(git describe linkml-conversion) + +master_mergebase=$(git merge-base $remote/master linkml-conversion) + git checkout linkml-auto-converted git merge -s ours --no-commit linkml-conversion @@ -19,18 +29,19 @@ git read-tree -m -u linkml-conversion # Poor man patch queue implementation # Edit this list if you want to merge or drop PRs branches to be patched with. # Order matters -branches_to_merge=( remove-discriminated-unions ) +branches_to_merge=( + master + remove-discriminated-unions +) -# determine remote name based on what remote linkml-conversion is -# tracked from which should correspond to our main repo! -remote=$(git config branch.linkml-conversion.remote) +echo "Applying patches from base $master_mergebase" for b in ${branches_to_merge[@]}; do b_ver=$(git describe "$remote"/$b) - echo "Applying branch $b with following differences" - git diff --stat "$remote"/master..."$remote"/$b + echo "Applying branch $b patch with following differences" + git diff --stat "$master_mergebase"..."$remote"/$b - git diff "$remote"/master..."$remote"/$b | patch -p1 + git diff "$master_mergebase"..."$remote"/$b | patch -p1 done hatch run linkml-auto-converted:2linkml From 76a9d771eacdf3da85a849d8f3951bdc12dc2844 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 18:12:13 -0400 Subject: [PATCH 19/90] Removing reporting branches - did not work --- tools/linkml_conversion | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index f2b8175b..cd94ea86 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -54,7 +54,4 @@ git add dandischema/models_linkml.py dandischema/models.yaml git mv dandischema/models.py dandischema/models_orig.py git mv dandischema/models_importstab.py dandischema/models.py -git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches - -branches merged: ${branches_to_merge[@]} -" -a --no-verify +git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches" -a --no-verify From c6172c4a7acdf0db4663d4a90663dc635d321a4f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 18:25:27 -0400 Subject: [PATCH 20/90] ENH: patch converted to pydantic model to strip dciteCOLO --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 316c271e..009ed3c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dependencies = [ ] [tool.hatch.envs.linkml-auto-converted.scripts] 2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' > dandischema/models.yaml" -2pydantic = "gen-pydantic --black dandischema/models.yaml > dandischema/models_linkml.py" +2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -e 's,dciteCOLON,,g' > dandischema/models_linkml.py" [tool.setuptools.packages.find] namespaces = true From e2d124cb93b9e2a046747be357d5d435d48c7813 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 20 Mar 2026 18:29:08 -0400 Subject: [PATCH 21/90] do pre-commit --- tools/linkml_conversion | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index cd94ea86..47cf02e1 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -54,4 +54,7 @@ git add dandischema/models_linkml.py dandischema/models.yaml git mv dandischema/models.py dandischema/models_orig.py git mv dandischema/models_importstab.py dandischema/models.py +# because we have pre-commit.ci doing that behind our back too! +pre-commit run --all || : + git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches" -a --no-verify From d8b1bea323a7c604cfe478a348f6fe1cd866a710 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 26 Mar 2026 11:56:49 -0700 Subject: [PATCH 22/90] feat: provide partial schema Provide a partial schema to be merged with the generated schema --- dandischema/models_merge.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 dandischema/models_merge.yaml diff --git a/dandischema/models_merge.yaml b/dandischema/models_merge.yaml new file mode 100644 index 00000000..17afaa9c --- /dev/null +++ b/dandischema/models_merge.yaml @@ -0,0 +1,7 @@ +# This file specifies a partial schema to be merged with the auto LinkML translation +# `dandischema.models`. It is a place to specify elements of the target dandischema +# LinkML schema that are not translatable from or not available in `dandischema.models` + +slots: + schemaKey: + designates_type: true From c0fbd02b95918d4aba4e593a4752689bb7f79cf4 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 26 Mar 2026 12:08:01 -0700 Subject: [PATCH 23/90] feat: enable `2linkml` to merge partial schema in `models_merge.yaml` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 009ed3c9..c73d200a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ dependencies = [ "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" ] [tool.hatch.envs.linkml-auto-converted.scripts] -2linkml = "pydantic2linkml -l INFO -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' > dandischema/models.yaml" +2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' > dandischema/models.yaml" 2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -e 's,dciteCOLON,,g' > dandischema/models_linkml.py" [tool.setuptools.packages.find] From ae7b30dd183ea4b882ac0fe30d84f7d346df31ca Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 31 Mar 2026 10:04:13 -0700 Subject: [PATCH 24/90] feat: further override `schemaKey` slot to make it a valid type designator `dandischema.models` use `schemaKey` in each Pydantic as a de facto type designator in LinkML. However, director translation to LinkML based on individual model's defintion is not possible. This override provided in the merge file completes the translation --- dandischema/models_merge.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dandischema/models_merge.yaml b/dandischema/models_merge.yaml index 17afaa9c..2f8357f2 100644 --- a/dandischema/models_merge.yaml +++ b/dandischema/models_merge.yaml @@ -4,4 +4,5 @@ slots: schemaKey: + ifabsent: null designates_type: true From b92119cc754bf54c0b2aa9bcda28b0d98e2c3acb Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 31 Mar 2026 16:35:25 -0700 Subject: [PATCH 25/90] feat: add YAML sanitization pipeline to `2linkml` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce `tools/sanitize-yaml` as an extensible stdin→stdout sanitization aggregator, and `tools/remove_impossible_slot_usage_notes.py` to strip pydantic2linkml-generated notes about impossible `schemaKey` slot usage entries. The `2linkml` script now pipes through `tools/sanitize-yaml` so future sanitization steps can be added there without modifying `pyproject.toml`. Both `kislyuk/yq` and `mikefarah/yq` were tried but neither could preserve the original YAML sequence indentation. The Python script uses `ruamel.yaml` instead, which supports round-trip YAML editing that preserves the original formatting. It is invoked via `hatch run linkml-auto-converted:python` to ensure `ruamel.yaml` from the hatch env is used. Note: ruamel.yaml currently introduces some unwanted line wrapping changes. Setting `yaml.width` was attempted numerous times but did not improve the situation, so it is left unset for now. Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 5 +-- tools/remove_impossible_slot_usage_notes.py | 36 +++++++++++++++++++++ tools/sanitize-yaml | 10 ++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100755 tools/remove_impossible_slot_usage_notes.py create mode 100755 tools/sanitize-yaml diff --git a/pyproject.toml b/pyproject.toml index c73d200a..1c367f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,10 +69,11 @@ python = "3.10" [tool.hatch.envs.linkml-auto-converted] dependencies = [ "black", # This allows `gen-pydantic` to format the generated Pydantic models with Black formatting - "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git" + "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git", + "ruamel.yaml" ] [tool.hatch.envs.linkml-auto-converted.scripts] -2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' > dandischema/models.yaml" +2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/sanitize-yaml > dandischema/models.yaml" 2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -e 's,dciteCOLON,,g' > dandischema/models_linkml.py" [tool.setuptools.packages.find] diff --git a/tools/remove_impossible_slot_usage_notes.py b/tools/remove_impossible_slot_usage_notes.py new file mode 100755 index 00000000..0cffc747 --- /dev/null +++ b/tools/remove_impossible_slot_usage_notes.py @@ -0,0 +1,36 @@ +# Remove all `notes` entries containing the "Impossible to generate slot usage +# entry for the schemaKey" message generated by pydantic2linkml from YAML content. +# Reads from stdin, writes to stdout. + +import sys + +from ruamel.yaml import YAML + +PATTERN = "pydantic2linkml: Impossible to generate slot usage entry for the schemaKey" + + +def process(obj): + if isinstance(obj, dict): + if "notes" in obj: + to_remove = [i for i, note in enumerate(obj["notes"]) if PATTERN in note] + for i in reversed(to_remove): + del obj["notes"][i] + if not obj["notes"]: + del obj["notes"] + for value in obj.values(): + process(value) + elif isinstance(obj, list): + for item in obj: + process(item) + + +try: + yaml = YAML() + yaml.preserve_quotes = True + data = yaml.load(sys.stdin) + process(data) + yaml.dump(data, sys.stdout) + print("remove_impossible_slot_usage_notes: done", file=sys.stderr) +except Exception as e: + print(f"remove_impossible_slot_usage_notes: failed — {e}", file=sys.stderr) + sys.exit(1) diff --git a/tools/sanitize-yaml b/tools/sanitize-yaml new file mode 100755 index 00000000..6136702a --- /dev/null +++ b/tools/sanitize-yaml @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Sanitize YAML content. +# Reads from stdin, writes to stdout. +# Add further sanitization steps here as additional pipes. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_impossible_slot_usage_notes.py" From 043c3d6e55e1b09a6f30ea6ade404260708f78d2 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 1 Apr 2026 09:49:41 -0700 Subject: [PATCH 26/90] feat: make the `schemaKey` slot required This ensures each data records provide a value for the `schemaKey`. This is solution to remedy the fact that `schemaKey` is a type designator and doesn't have a default value, unlike the setup in Pydantic models. --- dandischema/models_merge.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dandischema/models_merge.yaml b/dandischema/models_merge.yaml index 2f8357f2..0eff6cad 100644 --- a/dandischema/models_merge.yaml +++ b/dandischema/models_merge.yaml @@ -6,3 +6,4 @@ slots: schemaKey: ifabsent: null designates_type: true + required: true From 920997621006b822dfdf08d554acd6d88420d415 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 1 Apr 2026 09:57:14 -0700 Subject: [PATCH 27/90] fix: generalize and fix COLON sed pattern in 2pydantic script - Generalize the hardcoded `dciteCOLON` pattern to `[a-z]+COLON` to strip any lowercase-prefixed COLON artifact, not just `dciteCOLON` - Use `-E` flag with plain `+` instead of GNU-specific `\+` so the pattern works on both macOS (BSD sed) and Linux (GNU sed) Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1c367f5a..efd79aa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ dependencies = [ ] [tool.hatch.envs.linkml-auto-converted.scripts] 2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/sanitize-yaml > dandischema/models.yaml" -2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -e 's,dciteCOLON,,g' > dandischema/models_linkml.py" +2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -E -e 's,[a-z]+COLON,,g' > dandischema/models_linkml.py" [tool.setuptools.packages.find] namespaces = true From 9609e89b7c11a77e3cdbde924f72b4e674024bfb Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 5 Apr 2026 23:43:37 -0700 Subject: [PATCH 28/90] feat: add script to find models with mismatched schemaKey defaults Adds a diagnostic tool that identifies Pydantic models where the schemaKey field default differs from the containing class name. Co-Authored-By: Claude Opus 4.6 --- .../find_schemakey_mismatches.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tools/linkml_conversion_tools/find_schemakey_mismatches.py diff --git a/tools/linkml_conversion_tools/find_schemakey_mismatches.py b/tools/linkml_conversion_tools/find_schemakey_mismatches.py new file mode 100644 index 00000000..1fcb1f23 --- /dev/null +++ b/tools/linkml_conversion_tools/find_schemakey_mismatches.py @@ -0,0 +1,46 @@ +"""Find Pydantic models where schemaKey default != class name.""" + +import inspect +import sys + +from pydantic import BaseModel + +import dandischema.models as mod + +# The set of the names of classes where the schemaKey default doesn't match the class +# name in the last recorded inspection of this script +LAST_SCHEMAKEY_MISMATCHES = {"BareAsset", "PublishedAsset", "PublishedDandiset"} + +# The set of the names of classes where the schemaKey default doesn't match the class +# name +schemakey_mismatches: set[str] = set() + +for name, cls in inspect.getmembers(mod, inspect.isclass): + # Only consider Pydantic models + if not issubclass(cls, BaseModel) or cls is BaseModel: + continue + # Skip re-exported classes defined in other modules + if cls.__module__ != mod.__name__: + continue + + fields = cls.model_fields + # Only look at models that have a schemaKey field + if "schemaKey" not in fields: + continue + + field = fields["schemaKey"] + schema_key_default = field.default + # Flag models where the schemaKey default doesn't match the class name, + # including cases where the default is None (i.e. not explicitly set) + if schema_key_default != name: + print(f"{name}: schemaKey default = {schema_key_default!r}") + schemakey_mismatches.add(name) + +print("---------------") +if schemakey_mismatches == LAST_SCHEMAKEY_MISMATCHES: + print("No changes since last inspection.") +else: + print("WARNING: Changes since last inspection!") + print(f"New mismatches: {schemakey_mismatches - LAST_SCHEMAKEY_MISMATCHES}") + print(f"Resolved mismatches: {LAST_SCHEMAKEY_MISMATCHES - schemakey_mismatches}") + sys.exit(1) From e404946f24b5fbc6af92efb6edcd3025da4454b5 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 6 Apr 2026 18:48:25 -0700 Subject: [PATCH 29/90] feat: add note regarding mismatch of `schemaKey` default and model name This note is specified in the `models_merge.yaml` file to be added to the resulting LinkML translation of `dandischema.models`. --- dandischema/models_merge.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dandischema/models_merge.yaml b/dandischema/models_merge.yaml index 0eff6cad..6f59f771 100644 --- a/dandischema/models_merge.yaml +++ b/dandischema/models_merge.yaml @@ -7,3 +7,20 @@ slots: ifabsent: null designates_type: true required: true + +classes: + BareAsset: + notes: + - "The default of the `schemaKey` field in the corresponding Pydantic model in + `dandischema.models` is not the model's name. Adjustment to the inherited + `schemaKey` slot may be needed." + PublishedAsset: + notes: + - "The default of the `schemaKey` field in the corresponding Pydantic model in + `dandischema.models` is not the model's name. Adjustment to the inherited + `schemaKey` slot may be needed." + PublishedDandiset: + notes: + - "The default of the `schemaKey` field in the corresponding Pydantic model in + `dandischema.models` is not the model's name. Adjustment to the inherited + `schemaKey` slot may be needed." From 111e635c78cef56ebbd2166ecb5ae5a6de85755b Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 6 Apr 2026 19:09:24 -0700 Subject: [PATCH 30/90] chore: group linkml conversion helpers under tools/linkml_conversion_tools Move `remove_impossible_slot_usage_notes.py` and `sanitize-yaml` into `tools/linkml_conversion_tools/` alongside the other linkml conversion helpers, and update the `2linkml` script in `pyproject.toml` accordingly. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- .../remove_impossible_slot_usage_notes.py | 0 tools/{ => linkml_conversion_tools}/sanitize-yaml | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename tools/{ => linkml_conversion_tools}/remove_impossible_slot_usage_notes.py (100%) rename tools/{ => linkml_conversion_tools}/sanitize-yaml (100%) diff --git a/pyproject.toml b/pyproject.toml index efd79aa5..0785eadb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dependencies = [ "ruamel.yaml" ] [tool.hatch.envs.linkml-auto-converted.scripts] -2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/sanitize-yaml > dandischema/models.yaml" +2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/linkml_conversion_tools/sanitize-yaml > dandischema/models.yaml" 2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -E -e 's,[a-z]+COLON,,g' > dandischema/models_linkml.py" [tool.setuptools.packages.find] diff --git a/tools/remove_impossible_slot_usage_notes.py b/tools/linkml_conversion_tools/remove_impossible_slot_usage_notes.py similarity index 100% rename from tools/remove_impossible_slot_usage_notes.py rename to tools/linkml_conversion_tools/remove_impossible_slot_usage_notes.py diff --git a/tools/sanitize-yaml b/tools/linkml_conversion_tools/sanitize-yaml similarity index 100% rename from tools/sanitize-yaml rename to tools/linkml_conversion_tools/sanitize-yaml From 7dd492d58da4b3306ae9bf7f43dc58f07840490c Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Apr 2026 13:48:20 -0700 Subject: [PATCH 31/90] style: attach `MANUAL_NOTE` prefix to manually added notes --- dandischema/models_merge.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dandischema/models_merge.yaml b/dandischema/models_merge.yaml index 6f59f771..0bfb4f5f 100644 --- a/dandischema/models_merge.yaml +++ b/dandischema/models_merge.yaml @@ -11,16 +11,16 @@ slots: classes: BareAsset: notes: - - "The default of the `schemaKey` field in the corresponding Pydantic model in - `dandischema.models` is not the model's name. Adjustment to the inherited + - "MANUAL_NOTE: The default of the `schemaKey` field in the corresponding Pydantic + model in `dandischema.models` is not the model's name. Adjustment to the inherited `schemaKey` slot may be needed." PublishedAsset: notes: - - "The default of the `schemaKey` field in the corresponding Pydantic model in - `dandischema.models` is not the model's name. Adjustment to the inherited + - "MANUAL_NOTE: The default of the `schemaKey` field in the corresponding Pydantic + model in `dandischema.models` is not the model's name. Adjustment to the inherited `schemaKey` slot may be needed." PublishedDandiset: notes: - - "The default of the `schemaKey` field in the corresponding Pydantic model in - `dandischema.models` is not the model's name. Adjustment to the inherited + - "MANUAL_NOTE: The default of the `schemaKey` field in the corresponding Pydantic + model in `dandischema.models` is not the model's name. Adjustment to the inherited `schemaKey` slot may be needed." From c5903c9e240bfea48f8ee767f50906d5f284ddf2 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Apr 2026 14:33:19 -0700 Subject: [PATCH 32/90] refactor: generalize note-removal script to support multiple regex patterns Rename remove_impossible_slot_usage_notes.py to remove_notes_by_pattern.py and switch from a single substring match to a list of regex patterns matched via re.search, so additional note families can be stripped in the same pass. Co-Authored-By: Claude Opus 4.6 --- ...ge_notes.py => remove_notes_by_pattern.py} | 20 ++++++++++++------- tools/linkml_conversion_tools/sanitize-yaml | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) rename tools/linkml_conversion_tools/{remove_impossible_slot_usage_notes.py => remove_notes_by_pattern.py} (52%) mode change 100755 => 100644 diff --git a/tools/linkml_conversion_tools/remove_impossible_slot_usage_notes.py b/tools/linkml_conversion_tools/remove_notes_by_pattern.py old mode 100755 new mode 100644 similarity index 52% rename from tools/linkml_conversion_tools/remove_impossible_slot_usage_notes.py rename to tools/linkml_conversion_tools/remove_notes_by_pattern.py index 0cffc747..30d80d3f --- a/tools/linkml_conversion_tools/remove_impossible_slot_usage_notes.py +++ b/tools/linkml_conversion_tools/remove_notes_by_pattern.py @@ -1,18 +1,24 @@ -# Remove all `notes` entries containing the "Impossible to generate slot usage -# entry for the schemaKey" message generated by pydantic2linkml from YAML content. -# Reads from stdin, writes to stdout. +# Remove all `notes` entries matching any of the configured regex patterns +# from YAML content. Reads from stdin, writes to stdout. +import re import sys from ruamel.yaml import YAML -PATTERN = "pydantic2linkml: Impossible to generate slot usage entry for the schemaKey" +PATTERNS = [ + r"pydantic2linkml: Impossible to generate slot usage entry for the schemaKey", +] def process(obj): if isinstance(obj, dict): if "notes" in obj: - to_remove = [i for i, note in enumerate(obj["notes"]) if PATTERN in note] + to_remove = [ + i + for i, note in enumerate(obj["notes"]) + if any(re.search(p, note) for p in PATTERNS) + ] for i in reversed(to_remove): del obj["notes"][i] if not obj["notes"]: @@ -30,7 +36,7 @@ def process(obj): data = yaml.load(sys.stdin) process(data) yaml.dump(data, sys.stdout) - print("remove_impossible_slot_usage_notes: done", file=sys.stderr) + print("remove_notes_by_pattern: done", file=sys.stderr) except Exception as e: - print(f"remove_impossible_slot_usage_notes: failed — {e}", file=sys.stderr) + print(f"remove_notes_by_pattern: failed — {e}", file=sys.stderr) sys.exit(1) diff --git a/tools/linkml_conversion_tools/sanitize-yaml b/tools/linkml_conversion_tools/sanitize-yaml index 6136702a..662ab0c2 100755 --- a/tools/linkml_conversion_tools/sanitize-yaml +++ b/tools/linkml_conversion_tools/sanitize-yaml @@ -7,4 +7,4 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_impossible_slot_usage_notes.py" +hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_notes_by_pattern.py" From 0a210404a6c2d609d01f131d19d98a832b095ebd Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Apr 2026 15:06:49 -0700 Subject: [PATCH 33/90] feat: remove notes regarding translation max length constraints on strings The translation of the max length constraint on strings by specifying the constraint in the pattern of the string is the best expression available in LinkML. There is no direct expression in LinkML for a max length constraint for string range --- tools/linkml_conversion_tools/remove_notes_by_pattern.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/linkml_conversion_tools/remove_notes_by_pattern.py b/tools/linkml_conversion_tools/remove_notes_by_pattern.py index 30d80d3f..715916a6 100644 --- a/tools/linkml_conversion_tools/remove_notes_by_pattern.py +++ b/tools/linkml_conversion_tools/remove_notes_by_pattern.py @@ -8,6 +8,8 @@ PATTERNS = [ r"pydantic2linkml: Impossible to generate slot usage entry for the schemaKey", + r"LinkML does not have direct support for max length constraints\. " + r"The max length constraint .+ is incorporated into the pattern", ] From aeae21f0c67fe4494c85cba48fc20c81d6ea6d4a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 11 Apr 2026 15:04:45 -0400 Subject: [PATCH 34/90] doc: claude report on analysis for schemaKey divergence for some classes --- .../20260411-schemaKey-mismatch-report.md | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 docs/designs/20260411-schemaKey-mismatch-report.md diff --git a/docs/designs/20260411-schemaKey-mismatch-report.md b/docs/designs/20260411-schemaKey-mismatch-report.md new file mode 100644 index 00000000..5a53a342 --- /dev/null +++ b/docs/designs/20260411-schemaKey-mismatch-report.md @@ -0,0 +1,239 @@ +# schemaKey Mismatch Report + +## Related issues and PRs + +- [dandi/dandi-schema#389](https://github.com/dandi/dandi-schema/issues/389) -- + **Handle LinkML migration issue of `pydantic2linkml: Impossible to generate + slot usage entry for the`** (open). The parent issue for this investigation. + Documents 54 pydantic2linkml conversion errors, 41 of which concern + `schemaKey` slot usage. Includes the open TODO items about figuring out why + `schemaKey` differs from class name for Published/Bare models. +- [dandi/dandi-schema#388](https://github.com/dandi/dandi-schema/issues/388) -- + **Establish at least 8 issues for specific groups of problems in converted + linkml** (open). Umbrella issue cataloguing all pydantic2linkml translation + problems, grouped by category. +- [dandi/dandi-schema#385](https://github.com/dandi/dandi-schema/pull/385) -- + **Replace discriminated unions with simple unions in models** (open PR). + Removes `Field(discriminator="schemaKey")` in favor of plain `Union[...]`, + since LinkML has no discriminated union equivalent + ([dandi/pydantic2linkml#39](https://github.com/dandi/pydantic2linkml/issues/39)). + Each union member still has a distinct `schemaKey: Literal[...]`, so + Pydantic's smart union mode resolves correctly without an explicit + discriminator. Has a pending TODO to analyze effects on the Meditor. +- [dandi/dandi-schema#244](https://github.com/dandi/dandi-schema/issues/244) -- + **Use discriminated unions to improve validation errors** (closed). The + original issue that introduced discriminated unions on `schemaKey` to improve + validation error messages for `Union[Person, Organization]` contributor + fields. Now being reversed by #385 for LinkML compatibility. +- [dandi/dandi-schema#205](https://github.com/dandi/dandi-schema/issues/205) -- + **Overhaul models so that "unfinished" metadata can be represented without + cheating Pydantic** (open). Proposes separate draft vs published models so + that draft metadata does not need `model_construct()` to bypass validation. + Directly relevant to the Bare/Published model split that causes the schemaKey + mismatch. +- [dandi/dandi-schema#77](https://github.com/dandi/dandi-schema/pull/77) -- + **make schemaKey required and improve validation and migration functions** + (merged PR). Made `schemaKey` a required field in JSON Schema output and added + validation logic. +- [dandi/dandi-schema#68](https://github.com/dandi/dandi-schema/pull/68) -- + **ensure schemaKeys are set properly** (merged PR). Early work to set up + schemaKey defaults and validation across all models. +- [dandi/dandi-schema#13](https://github.com/dandi/dandi-schema/pull/13) -- + **Fix/schemakey metaclass** (merged PR). Original implementation of the + schemaKey metaclass and the `enum` -> `const` conversion in JSON Schema + export. + +## Summary + +Three pydantic model classes in `dandischema/models.py` have `schemaKey` values +that do not match their class name: + +| Class | schemaKey | Expected | +|--------------------|-------------|-------------------| +| `BareAsset` | `"Asset"` | `"BareAsset"` | +| `PublishedDandiset` | `"Dandiset"` | `"PublishedDandiset"` | +| `PublishedAsset` | `"Asset"` | `"PublishedAsset"` | + +(`Asset` inherits `"Asset"` from `BareAsset` and does match its class name, so +it is not a mismatch.) + +## Why it was done this way + +### 1. Deliberate validator logic + +The `ensure_schemakey` validator (`models.py:570-582`) has explicit special-case +logic that compensates for the mismatch: + +```python +if "Published" in cls.__name__: + tempval = "Published" + tempval # "Dandiset" -> "PublishedDandiset" +elif "BareAsset" == cls.__name__: + tempval = "Bare" + tempval # "Asset" -> "BareAsset" +if tempval != cls.__name__: + raise ValueError(...) +``` + +This proves the mismatch was intentional, not accidental. + +### 2. Conceptual collapsing in JSON Schema + +When exporting to JSON Schema (`__get_pydantic_json_schema__`, line 659-664), +`schemaKey` is emitted as a `const`: + +```python +if prop == "schemaKey": + if "enum" in value and len(value["enum"]) == 1: + value["const"] = value["enum"][0] + del value["enum"] + else: + value["const"] = value["default"] +``` + +This means: +- `BareAsset`, `Asset`, `PublishedAsset` all produce `"schemaKey": {"const": "Asset"}` +- `Dandiset`, `PublishedDandiset` both produce `"schemaKey": {"const": "Dandiset"}` + +The design intent was that from a JSON Schema / API consumer perspective, there +are only two top-level entity kinds: **Asset** and **Dandiset**. The +Bare/Published distinctions are Python model hierarchy implementation details. + +## How schemaKey is used + +### dandi-archive backend (`dandiapi/`) + +1. **Validation** (`dandiapi/api/services/metadata/__init__.py`): The archive + explicitly calls `validate(metadata, schema_key='PublishedAsset')` and + `validate(..., schema_key='PublishedDandiset')` -- passing the **class name** + as `schema_key`, not the schemaKey value. The `validate()` function in + dandischema (`metadata.py:328`) uses the `schema_key` parameter to do + `getattr(models, schema_key)` to look up the pydantic class directly. So the + archive backend **bypasses schemaKey entirely** for choosing which model to + validate against. + +2. **Schema endpoint** (`dandiapi/api/views/schema.py`): Exposes JSON schemas + for `Dandiset`, `Asset`, `PublishedDandiset`, `PublishedAsset` via + `?model=`. The mapping uses `__name__` (class name), not + `schemaKey`. So again, the model identity is carried by the Python class name, + not by `schemaKey`. + +3. **Default metadata construction** (`dandiapi/api/services/version/metadata.py`, + `dandiapi/api/models/version.py`): When constructing default metadata dicts, + the archive hardcodes `'schemaKey': 'Dandiset'` and `'schemaKey': 'Asset'`. + It never writes `'PublishedDandiset'`, `'PublishedAsset'`, or `'BareAsset'` + as schemaKey values. + +4. **Metadata stripping** (`dandiapi/api/models/version.py:196-203`): The + `strip_metadata` method explicitly strips `schemaKey` from `access` sub-objects, + treating it as a computed/server-controlled field. + +### dandi-archive frontend (Meditor) + +1. **Schema fetching** (`web/src/stores/dandiset.ts:112-122`): The Meditor + fetches the JSON Schema from the API endpoint + (`/api/schemas/?model=Dandiset`). This returns the `Dandiset` model's JSON + Schema where `schemaKey` has `"const": "Dandiset"`. + +2. **Meditor types** (`web/src/components/Meditor/types.ts:41-48`): The type + `SchemaKeyPropertiesIntersection` expects schemas to have: + ```typescript + schemaKey: { + type: 'string'; + const: string; // expects a const value + }; + ``` + The Meditor treats `schemaKey` as a read-only const field. It does **not** + use the schemaKey value to dispatch between different model types; it relies + on the JSON Schema structure itself. + +3. **Discriminator usage in Vue components** + (`web/src/components/DLP/OverviewTab.vue`, `web/src/utils/cff.ts`): The + frontend uses `schemaKey` to discriminate between `Person` and `Organization` + in contributor lists, but this is among truly distinct types -- not among + Bare/Published variants. + +4. **VJSF rendering**: The Meditor uses VJSF (Vue JSON Schema Forms) which + renders forms from JSON Schema. The `schemaKey` field's `const` constraint + means it appears as a non-editable fixed value. The Meditor's `utils.ts` + validates using Ajv against the schema, and the `const: "Dandiset"` constraint + means submitted metadata must have `schemaKey: "Dandiset"`. + +### dandi-cli + +- `dandi-cli` uses `schemaKey` as a **discriminator** for pydantic tagged unions + (e.g., `Field(discriminator="schemaKey")` at `models.py:1284` for + `Union[Person, Organization, Software, Agent]`). +- In tests, `BareAsset` instances are constructed with `schemaKey="Asset"`. +- The `schemaKey` value `"Session"` vs `"Activity"` is used to filter + `wasGeneratedBy` entries (`test_metadata.py:528`). + +### dandischema validation (`metadata.py`) + +- `SCHEMA_MAP` maps class names to JSON Schema files: `"Dandiset"` -> + `"dandiset.json"`, `"PublishedDandiset"` -> `"published-dandiset.json"`, etc. +- `validate()` accepts an explicit `schema_key` parameter (class name). If not + provided, it falls back to `obj.get("schemaKey")` -- and since stored objects + have `schemaKey: "Dandiset"` (not `"PublishedDandiset"`), this fallback would + select the wrong (base) model for Published variants. +- This is why dandi-archive always passes `schema_key='PublishedDandiset'` + explicitly. + +## Impact of making schemaKey match model name + +### What would change + +If `BareAsset.schemaKey` becomes `"BareAsset"`, `PublishedDandiset.schemaKey` +becomes `"PublishedDandiset"`, and `PublishedAsset.schemaKey` becomes +`"PublishedAsset"`: + +1. **dandischema `ensure_schemakey` validator**: The special-case prefix logic + can be removed -- each class simply checks `val == cls.__name__`. + +2. **dandischema `validate()` fallback**: The `obj.get("schemaKey")` fallback + path would now correctly resolve to the right model for Published variants. + +3. **JSON Schema output**: Each model would emit its own unique `const` value, + enabling true type discrimination in JSON Schema. + +4. **dandi-archive backend**: Hardcoded `'schemaKey': 'Dandiset'` and + `'schemaKey': 'Asset'` in metadata construction would need to stay as-is for + draft versions (which use `Dandiset`/`Asset` models) but + `PublishedDandiset`/`PublishedAsset` validation already passes `schema_key` + explicitly so no change needed there. + +5. **dandi-archive Meditor**: The schema endpoint returns the `Dandiset` model's + schema (not `PublishedDandiset`), so `const: "Dandiset"` stays the same for + the editor. No Meditor change needed. + +6. **Existing published metadata**: All published dandiset metadata in the + database has `schemaKey: "Dandiset"` and `schemaKey: "Asset"`. A migration + would be needed to update existing records if we want consistency, or the + Published models must accept both old and new values during a transition + period. + +### Benefits for LinkML + +- `schemaKey` can serve as a proper **type designator** for deserialization +- Satisfies LinkML's **monotonic slot constraint** (each class gets a unique + const, no conflicting overrides in slot_usage) +- Enables correct round-tripping: serialize -> schemaKey -> deserialize to the + right class + +### Risks + +- **Data migration**: Existing records in the archive database and published + metadata files contain `schemaKey: "Dandiset"` / `"Asset"` for what are + actually `PublishedDandiset` / `PublishedAsset` instances. Any consumer that + matches on the exact string would need updating. +- **Schema version boundary**: This is a semantic change that ideally coincides + with a schema version bump. + +## Recommendation + +Making `schemaKey` match the model name is the correct path forward for the +LinkML migration. The current mismatch exists purely as a legacy design choice +that treats Bare/Published as invisible variants. In practice, the archive +backend already works around this by passing explicit `schema_key` parameters. +The Meditor is unaffected since it only deals with draft `Dandiset` metadata. + +The change should be coordinated with a schema version bump and a data migration +plan for existing published records. From 2c38801e0cd626427a3edc1d2363c7d60c601bd4 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 14 Apr 2026 23:50:30 -0700 Subject: [PATCH 35/90] feat: add script to remove schemaKey entries from slot_usage in class definitions Co-Authored-By: Claude Sonnet 4.6 --- .../remove_slot_usage_schemakey.py | 38 +++++++++++++++++++ tools/linkml_conversion_tools/sanitize-yaml | 3 +- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tools/linkml_conversion_tools/remove_slot_usage_schemakey.py diff --git a/tools/linkml_conversion_tools/remove_slot_usage_schemakey.py b/tools/linkml_conversion_tools/remove_slot_usage_schemakey.py new file mode 100644 index 00000000..2c7a2df9 --- /dev/null +++ b/tools/linkml_conversion_tools/remove_slot_usage_schemakey.py @@ -0,0 +1,38 @@ +# Remove all `schemaKey` entries inside `slot_usage` blocks of class definitions +# from YAML content. Reads from stdin, writes to stdout. + +import sys + +from ruamel.yaml import YAML + + +def process(data_): + for class_def in data_.get("classes", {}).values(): + if not isinstance(class_def, dict): + raise ValueError( + f"Expected class definition to be a dict, got {type(class_def)}" + ) + if "slot_usage" in class_def: + slot_usage = class_def["slot_usage"] + if not isinstance(slot_usage, dict): + raise ValueError( + f"Expected slot_usage to be a dict, got {type(class_def['slot_usage'])}" + ) + if "schemaKey" in slot_usage: + del slot_usage["schemaKey"] + + # If slot_usage is now empty, remove it entirely + if not slot_usage: + del class_def["slot_usage"] + + +try: + yaml = YAML() + yaml.preserve_quotes = True + data = yaml.load(sys.stdin) + process(data) + yaml.dump(data, sys.stdout) + print("remove_slot_usage_schemakey: done", file=sys.stderr) +except Exception as e: + print(f"remove_slot_usage_schemakey: failed — {e}", file=sys.stderr) + sys.exit(1) diff --git a/tools/linkml_conversion_tools/sanitize-yaml b/tools/linkml_conversion_tools/sanitize-yaml index 662ab0c2..4cc07191 100755 --- a/tools/linkml_conversion_tools/sanitize-yaml +++ b/tools/linkml_conversion_tools/sanitize-yaml @@ -7,4 +7,5 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_notes_by_pattern.py" +hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_notes_by_pattern.py" | +hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_slot_usage_schemakey.py" From 7773f7552af05d80ea26cbf62bd9fe35cbd84521 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 16 Apr 2026 14:53:09 -0700 Subject: [PATCH 36/90] feat: add script to sort LicenseType permissible values alphabetically Ensures `enums.LicenseType.permissible_values` entries are sorted by key for stable, readable diffs in the auto-converted LinkML YAML output. Co-Authored-By: Claude Opus 4.6 --- tools/linkml_conversion_tools/sanitize-yaml | 3 +- .../sort_license_type_permissible_values.py | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tools/linkml_conversion_tools/sort_license_type_permissible_values.py diff --git a/tools/linkml_conversion_tools/sanitize-yaml b/tools/linkml_conversion_tools/sanitize-yaml index 4cc07191..3fcf98eb 100755 --- a/tools/linkml_conversion_tools/sanitize-yaml +++ b/tools/linkml_conversion_tools/sanitize-yaml @@ -8,4 +8,5 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_notes_by_pattern.py" | -hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_slot_usage_schemakey.py" +hatch run linkml-auto-converted:python "$SCRIPT_DIR/remove_slot_usage_schemakey.py" | +hatch run linkml-auto-converted:python "$SCRIPT_DIR/sort_license_type_permissible_values.py" diff --git a/tools/linkml_conversion_tools/sort_license_type_permissible_values.py b/tools/linkml_conversion_tools/sort_license_type_permissible_values.py new file mode 100644 index 00000000..cfae7a9d --- /dev/null +++ b/tools/linkml_conversion_tools/sort_license_type_permissible_values.py @@ -0,0 +1,32 @@ +# Sort entries in `enums.LicenseType.permissible_values` alphabetically by +# key. Reads from stdin, writes to stdout. + +import sys + +from ruamel.yaml import YAML + + +def process(data_): + match data_: + case { + "enums": {"LicenseType": {"permissible_values": dict(permissible_values)}} + }: + sorted_items = sorted(permissible_values.items()) + permissible_values.clear() + permissible_values.update(sorted_items) + case _: + raise ValueError( + "Expected `enums.LicenseType.permissible_values` to be a mapping" + ) + + +try: + yaml = YAML() + yaml.preserve_quotes = True + data = yaml.load(sys.stdin) + process(data) + yaml.dump(data, sys.stdout) + print("sort_license_type_permissible_values: done", file=sys.stderr) +except Exception as e: + print(f"sort_license_type_permissible_values: failed — {e}", file=sys.stderr) + sys.exit(1) From b688b3b76bde71eb9cfeb5078d1799c985456e29 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Apr 2026 16:28:07 -0400 Subject: [PATCH 37/90] Import also original DANDI_NSKEY and get_schema_version into models.py ATM needed by genschemata helper. potentially we could just patch there. --- dandischema/models_importstab.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dandischema/models_importstab.py b/dandischema/models_importstab.py index 6815ca49..2b70733c 100644 --- a/dandischema/models_importstab.py +++ b/dandischema/models_importstab.py @@ -1,5 +1,9 @@ from .models_linkml import * # noqa: F401,F403 -from .models_orig import DANDI_INSTANCE_URL_PATTERN # noqa: F401 +from .models_orig import ( # noqa: F401 + DANDI_INSTANCE_URL_PATTERN, + DANDI_NSKEY, + get_schema_version, +) # TODO: temporary imports of consts etc which might need to be 'redone' # so we do not duplicate them From db7be48bc2180620b94c5ab6144ae18ada761f2d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Apr 2026 16:29:12 -0400 Subject: [PATCH 38/90] ENH: add 2 hatch specs for conversions into jsonschema for comparisons --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0785eadb..6188f502 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,8 @@ dependencies = [ [tool.hatch.envs.linkml-auto-converted.scripts] 2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/linkml_conversion_tools/sanitize-yaml > dandischema/models.yaml" 2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -E -e 's,[a-z]+COLON,,g' > dandischema/models_linkml.py" +2json = "d=dandischema/models_linkml; rm -rf $d && mkdir -p $d && for t in Dandiset Asset PublishedDandiset PublishedAsset; do gen-json-schema -t $t dandischema/models.yaml >| $d/$t.json; done" +pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && python tools/pubschemata.py $d && mv $d/*/*json $d/" [tool.setuptools.packages.find] namespaces = true From 3b22cd78dcb7a39929d02ef28d0f02ad273e2fff Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Apr 2026 16:40:39 -0400 Subject: [PATCH 39/90] Do also conversions to jsonschema for comparisons --- tools/linkml_conversion | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index 47cf02e1..f2801cd7 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -54,6 +54,14 @@ git add dandischema/models_linkml.py dandischema/models.yaml git mv dandischema/models.py dandischema/models_orig.py git mv dandischema/models_importstab.py dandischema/models.py +# for comparisons sake +hatch run linkml-auto-converted:2json +hatch run linkml-auto-converted:pydantic2json + +git add dandischema/models_linkml +git add dandischema/models_pydantic + + # because we have pre-commit.ci doing that behind our back too! pre-commit run --all || : From 14c5177c8c354c75d5a0f4c012e1040f00346a4f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Apr 2026 17:09:20 -0400 Subject: [PATCH 40/90] Harmonize converted files --- tools/linkml_conversion | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index f2801cd7..07dedcfe 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -54,11 +54,20 @@ git add dandischema/models_linkml.py dandischema/models.yaml git mv dandischema/models.py dandischema/models_orig.py git mv dandischema/models_importstab.py dandischema/models.py -# for comparisons sake +# for comparisons sake -- from linkml to jsonschema hatch run linkml-auto-converted:2json -hatch run linkml-auto-converted:pydantic2json - +# Rename for consistency +( + cd dandischema/models_linkml + for f in *; do + nf=$(echo $f | tr '[A-Z]' '[a-z]' | sed -e 's,published,published-,g') + mv "$f" "$nf" + done +) git add dandischema/models_linkml + +hatch run linkml-auto-converted:pydantic2json +rmdir dandischema/models_pydantic/[0-9]* git add dandischema/models_pydantic From 05576e93e7d3637fe6063c4d81fb7cb048623f80 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Apr 2026 17:22:39 -0400 Subject: [PATCH 41/90] Split into two conversion script so we could play with linkml model quickly --- tools/linkml_conversion | 25 +++++------------------ tools/linkml_conversion_fromlinkml | 32 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 20 deletions(-) create mode 100755 tools/linkml_conversion_fromlinkml diff --git a/tools/linkml_conversion b/tools/linkml_conversion index 07dedcfe..7d9945f5 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -45,31 +45,16 @@ for b in ${branches_to_merge[@]}; do done hatch run linkml-auto-converted:2linkml -hatch run linkml-auto-converted:2pydantic - -# add (re)generated files -git add dandischema/models_linkml.py dandischema/models.yaml +git add dandischema/models.yaml +# The renaming dance in preparation to accept converted pyndatic model # add our stab and rename original models.py git mv dandischema/models.py dandischema/models_orig.py git mv dandischema/models_importstab.py dandischema/models.py -# for comparisons sake -- from linkml to jsonschema -hatch run linkml-auto-converted:2json -# Rename for consistency -( - cd dandischema/models_linkml - for f in *; do - nf=$(echo $f | tr '[A-Z]' '[a-z]' | sed -e 's,published,published-,g') - mv "$f" "$nf" - done -) -git add dandischema/models_linkml - -hatch run linkml-auto-converted:pydantic2json -rmdir dandischema/models_pydantic/[0-9]* -git add dandischema/models_pydantic - +# Placed separately so we could play with changing linkml model and redoing the +# conversion(s) and seeing the effect +$(dirname $0)/linkml_conversion_fromlinkml # because we have pre-commit.ci doing that behind our back too! pre-commit run --all || : diff --git a/tools/linkml_conversion_fromlinkml b/tools/linkml_conversion_fromlinkml new file mode 100755 index 00000000..cac8b889 --- /dev/null +++ b/tools/linkml_conversion_fromlinkml @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Script which will converts from linkml model into all other serializations for comparisons +# + +set -eu + +# 2 pydantic from linkml + +hatch run linkml-auto-converted:2pydantic +# add (re)generated files +git add dandischema/models_linkml.py + +# for comparisons sake -- from linkml to jsonschema +hatch run linkml-auto-converted:2json +# Rename for consistency +( + cd dandischema/models_linkml + for f in *; do + nf=$(echo $f | tr '[A-Z]' '[a-z]' | sed -e 's,published,published-,g') + mv "$f" "$nf" + done +) +git add dandischema/models_linkml + +hatch run linkml-auto-converted:pydantic2json +rmdir dandischema/models_pydantic/[0-9]* +git add dandischema/models_pydantic + + +# because we have pre-commit.ci doing that behind our back too! +pre-commit run --all || : From 33dd458cd1a5f4a78230c927e45a05ec5612b613 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 17 Apr 2026 17:27:21 -0400 Subject: [PATCH 42/90] Stage after all pre-commit changes --- tools/linkml_conversion_fromlinkml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/linkml_conversion_fromlinkml b/tools/linkml_conversion_fromlinkml index cac8b889..40a9f54c 100755 --- a/tools/linkml_conversion_fromlinkml +++ b/tools/linkml_conversion_fromlinkml @@ -21,7 +21,6 @@ hatch run linkml-auto-converted:2json mv "$f" "$nf" done ) -git add dandischema/models_linkml hatch run linkml-auto-converted:pydantic2json rmdir dandischema/models_pydantic/[0-9]* @@ -30,3 +29,5 @@ git add dandischema/models_pydantic # because we have pre-commit.ci doing that behind our back too! pre-commit run --all || : + +git add dandischema/models_linkml dandischema/models_linkml.py From 982491a3353452e6ed06acdb1befcbd73de5a24b Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 20 Apr 2026 18:15:35 -0700 Subject: [PATCH 43/90] Pin linkml and linkml-runtime in linkml-auto-converted hatch env Lock to the versions currently resolved via pydantic2linkml to prevent unintended changes when new linkml versions are released. Co-Authored-By: Claude Opus 4.7 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6188f502..1697fd8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,8 @@ python = "3.10" [tool.hatch.envs.linkml-auto-converted] dependencies = [ "black", # This allows `gen-pydantic` to format the generated Pydantic models with Black formatting + "linkml==1.10.0", + "linkml-runtime==1.10.0", "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git", "ruamel.yaml" ] From 4ec6cda3365baae48d0f2749e8d34f139e6dd9c6 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 20 Apr 2026 18:44:27 -0700 Subject: [PATCH 44/90] feat: update notes regarding length restrictions to be removed --- tools/linkml_conversion_tools/remove_notes_by_pattern.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/linkml_conversion_tools/remove_notes_by_pattern.py b/tools/linkml_conversion_tools/remove_notes_by_pattern.py index 715916a6..113c0ec1 100644 --- a/tools/linkml_conversion_tools/remove_notes_by_pattern.py +++ b/tools/linkml_conversion_tools/remove_notes_by_pattern.py @@ -8,8 +8,8 @@ PATTERNS = [ r"pydantic2linkml: Impossible to generate slot usage entry for the schemaKey", - r"LinkML does not have direct support for max length constraints\. " - r"The max length constraint .+ is incorporated into the pattern", + r"Length constraint of min_length=[a-zA-Z0-9]+, max_length=[a-zA-Z0-9]+ expressed " + r"as a pattern entry", ] From 8737dc9476a2130b28942f1379ae0a4cab7491b5 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 26 Apr 2026 14:11:11 -0700 Subject: [PATCH 45/90] Add scripts for LinkML validation of DANDI Archive metadata These three Typer scripts under .claude/skills/dandi-linkml-validation-report/scripts/ make up a reproducible pipeline for validating dandiset metadata against dandischema/models.yaml using the linkml-validate CLI: - fetch_metadata.py: download raw metadata for the draft and every published version of every dandiset on a DANDI instance, plus a small info.json per version capturing schemaVersion / status / modified. - validate_metadata.py: shell out to linkml-validate per version with the right target class (Dandiset for drafts, PublishedDandiset otherwise), writing validation.txt, validation.json and SUMMARY.md alongside each metadata.json. - generate_report.py: aggregate the per-version JSON records into a top-level REPORT.md grouped by class x schemaVersion, with most-common problem patterns and links to per-version summaries. The dandi client is added as a dependency of the linkml-auto-converted hatch env so the fetch script can run inside it. --- .../scripts/fetch_metadata.py | 202 +++++++++++ .../scripts/generate_report.py | 336 ++++++++++++++++++ .../scripts/validate_metadata.py | 251 +++++++++++++ pyproject.toml | 1 + 4 files changed, 790 insertions(+) create mode 100644 .claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py create mode 100644 .claude/skills/dandi-linkml-validation-report/scripts/generate_report.py create mode 100644 .claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py new file mode 100644 index 00000000..814ba02c --- /dev/null +++ b/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Download raw ``Dandiset`` metadata from a DANDI Archive instance. + +For every dandiset on the chosen instance this script writes the raw +metadata of the draft version *and* of every published version to:: + + ///metadata.json + +Each version directory also gets an ``info.json`` with the few fields +the downstream validation and report scripts need: + + { + "dandiset_id": "000003", + "version": "0.230629.1955", # or "draft" + "is_published": true, # false for the draft version + "status": "VALID", # archive-side status + "modified": "2023-06-29T...", # ISO 8601 or null + "schema_version": "0.6.4" # raw["schemaVersion"], may be null + } + +Re-running is safe: versions whose ``metadata.json`` already exists are +skipped unless ``--refresh`` is passed. This makes it easy to resume +after a network blip or to top up a previously-fetched directory with +newly published versions. + +Example +------- +:: + + python fetch_metadata.py linkml-validation-reports//data +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Optional + +from dandi.dandiapi import DandiAPIClient, RemoteDandiset +import typer + +logger = logging.getLogger("fetch_metadata") + +app = typer.Typer(add_completion=False, help=__doc__.splitlines()[0]) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_json(path: Path, data: object) -> None: + """Write ``data`` to ``path`` as pretty-printed JSON, creating parent + directories as needed. + + ``default=str`` lets us serialise ``datetime`` objects without having + to convert them by hand. + """ + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, default=str) + "\n") + + +def _fetch_version( + dandiset: RemoteDandiset, + version_id: str, + *, + is_published: bool, + version_dir: Path, + refresh: bool, +) -> None: + """Fetch ``metadata.json`` and ``info.json`` for one version. + + Parameters + ---------- + dandiset: + The ``RemoteDandiset`` object returned by the DANDI client. + version_id: + Either the published version identifier (e.g. ``"0.230629.1955"``) + or the literal string ``"draft"``. + is_published: + ``False`` for the draft version, ``True`` for any published + version. Persisted into ``info.json`` so the validator can pick + the right target class without re-querying the archive. + version_dir: + Destination directory; created if it does not yet exist. + refresh: + If ``False`` and the destination already contains both + ``metadata.json`` and ``info.json``, do nothing. + """ + metadata_file = version_dir / "metadata.json" + info_file = version_dir / "info.json" + if metadata_file.exists() and info_file.exists() and not refresh: + logger.debug("skip %s/%s (already downloaded)", dandiset.identifier, version_id) + return + + # ``for_version`` returns a fresh handle bound to the requested version, + # which is what ``get_raw_metadata`` and ``get_version`` need to operate on. + ds_at_version = dandiset.for_version(version_id) + raw = ds_at_version.get_raw_metadata() + version_info = ds_at_version.get_version(version_id) + + info = { + "dandiset_id": dandiset.identifier, + "version": version_id, + "is_published": is_published, + # ``status`` is an enum-like object on the client; ``.value`` if + # available, otherwise fall back to ``str()``. + "status": getattr(version_info.status, "value", str(version_info.status)), + "modified": ( + version_info.modified.isoformat() + if version_info.modified is not None + else None + ), + # The raw metadata's ``schemaVersion`` field is the dimension we + # want to group by in the top-level report, so capture it now. + "schema_version": raw.get("schemaVersion"), + } + _write_json(metadata_file, raw) + _write_json(info_file, info) + logger.info("fetched %s/%s", dandiset.identifier, version_id) + + +# --------------------------------------------------------------------------- +# Typer entry point +# --------------------------------------------------------------------------- + + +@app.command() +def main( + output_dir: Path = typer.Argument( + ..., + help="Directory under which //metadata.json " + "files will be written.", + ), + dandi_instance: str = typer.Option( + "dandi", + "--dandi-instance", + "-i", + help="DANDI server instance name as understood by `DandiAPIClient." + "for_dandi_instance`", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Re-download versions whose metadata is already on disk.", + ), + limit: Optional[int] = typer.Option( + None, + "--limit", + help="Process at most N dandisets (useful for smoke tests).", + ), + log_level: str = typer.Option("INFO", "--log-level", "-l"), +) -> None: + """Fetch metadata for all dandisets (draft + published versions).""" + logging.basicConfig( + format="[%(asctime)s] %(levelname)s %(name)s: %(message)s", + level=getattr(logging, log_level.upper()), + ) + output_dir.mkdir(parents=True, exist_ok=True) + + with DandiAPIClient.for_dandi_instance(dandi_instance) as client: + for i, dandiset in enumerate(client.get_dandisets(draft=True, order="id")): + if limit is not None and i >= limit: + break + dandiset_id = dandiset.identifier + dandiset_dir = output_dir / dandiset_id + logger.info("processing %s", dandiset_id) + + # The draft version always exists and is what new edits land on, + # so fetch it first. + try: + _fetch_version( + dandiset, + dandiset.draft_version.identifier, + is_published=False, + version_dir=dandiset_dir / "draft", + refresh=refresh, + ) + except Exception as e: + # Never let a single dandiset blow up the whole run. + logger.error("failed draft of %s: %s", dandiset_id, e) + + # Then walk every published version (skipping the draft, which + # ``get_versions`` also yields). + for v in dandiset.get_versions(): + if v.identifier == "draft": + continue + try: + _fetch_version( + dandiset, + v.identifier, + is_published=True, + version_dir=dandiset_dir / v.identifier, + refresh=refresh, + ) + except Exception as e: + logger.error("failed %s/%s: %s", dandiset_id, v.identifier, e) + + +if __name__ == "__main__": + app() diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py new file mode 100644 index 00000000..4e0c4631 --- /dev/null +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Generate the top-level Markdown validation report. + +The report directory is expected to look like:: + + linkml-validation-reports// + ├── REPORT.md <-- written by this script + └── data/ + ├── 000003/ + │ ├── draft/ + │ │ ├── metadata.json + │ │ ├── info.json + │ │ ├── validation.json + │ │ ├── validation.txt + │ │ └── SUMMARY.md + │ └── 0.230629.1955/... + └── 000004/... + +For each ``validation.json`` produced by ``validate_metadata.py``, this +script aggregates results and writes a single ``REPORT.md`` at the top +of the report directory. + +The report contains: + + * Header with the ``linkml-auto-converted`` commit hash and commit + date (passed via ``--commit-hash`` / ``--commit-date`` so the + script doesn't need to know which branch is in play). + + * Per-bucket summary tables, where a "bucket" is the cross product + of *target class* (``Dandiset`` for drafts, ``PublishedDandiset`` + for published versions) and *schemaVersion* of the raw metadata. + + * For every bucket, the most common error patterns (after stripping + the per-file path prefix added by ``linkml-validate``) so the + reader can spot systemic issues. + + * Per-bucket index linking each version to its per-version + ``SUMMARY.md``, so the report reads naturally on GitHub or any + static markdown viewer — no HTTP server needed. + +Example +------- +:: + + python generate_report.py linkml-validation-reports/ \\ + --commit-hash 54085828c72b69f3b9933dbd288114a9d074ed46 \\ + --commit-date 2026-04-20T18:47:47-07:00 +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +import json +import logging +from pathlib import Path +import re + +import typer + +logger = logging.getLogger("generate_report") + +app = typer.Typer(add_completion=False, help=__doc__.splitlines()[0]) + + +# --------------------------------------------------------------------------- +# Loading + grouping +# --------------------------------------------------------------------------- + + +# A line emitted by ``linkml-validate`` looks like:: +# +# [ERROR] [path/to/metadata.json/0] in /json/pointer +# +# The bracketed source-file segment is per-version noise that prevents +# similar errors across dandisets from grouping together. This regex +# strips it so we can count error patterns meaningfully. +_SOURCE_PREFIX_RE = re.compile(r"^(\[[A-Z]+\])\s+\[[^\]]+\]\s*") + + +def _normalise_problem(line: str) -> str: + """Drop the per-file source prefix from a problem line. + + Turns ``"[ERROR] [.../000003/.../metadata.json/0] foo in /bar"`` into + ``"[ERROR] foo in /bar"``, which groups across dandisets. + """ + return _SOURCE_PREFIX_RE.sub(r"\1 ", line, count=1) + + +def _load_records(data_dir: Path) -> list[dict]: + """Load every ``validation.json`` under ``data_dir``. + + Each record is augmented with the relative path to its per-version + ``SUMMARY.md`` so the report can link directly to it. + """ + records: list[dict] = [] + for vj in sorted(data_dir.glob("*/*/validation.json")): + try: + rec = json.loads(vj.read_text()) + except json.JSONDecodeError: + logger.warning("skipping unreadable %s", vj) + continue + # Path to per-version SUMMARY.md, relative to REPORT.md (which + # sits one level above ``data_dir``). + rec["_summary_link"] = ( + f"data/{vj.parent.parent.name}/{vj.parent.name}/SUMMARY.md" + ) + records.append(rec) + return records + + +def _bucket_key(rec: dict) -> tuple[str, str]: + """Return the ``(class, schema_version)`` bucket key for a record. + + ``schema_version`` may legitimately be missing from very old + metadata; we fold those into a synthetic ``""`` bucket so + they're still surfaced rather than dropped. + """ + sv = rec.get("schema_version") or "" + return rec["target_class"], sv + + +# --------------------------------------------------------------------------- +# Markdown rendering +# --------------------------------------------------------------------------- + + +def _render_bucket( + fh, + title: str, + records: list[dict], + *, + top_n_patterns: int, +) -> None: + """Render one bucket section to ``fh``. + + Emits: + * a one-line headline counting versions / valid / failing, + * a "top error patterns" list, + * a table indexing every version with a link to its + per-version ``SUMMARY.md``. + """ + n_total = len(records) + n_valid = sum(1 for r in records if r["problem_count"] == 0) + n_failing = n_total - n_valid + + fh.write(f"### {title}\n\n") + fh.write( + f"- **Versions:** {n_total} • " + f"**Valid:** {n_valid} • **With problems:** {n_failing}\n\n" + ) + + # --- Top error patterns within this bucket. --- + pattern_counter: Counter[str] = Counter() + for r in records: + for problem in r.get("problems", []): + pattern_counter[_normalise_problem(problem)] += 1 + if pattern_counter: + fh.write(f"**Top {top_n_patterns} problem patterns:**\n\n") + for pattern, count in pattern_counter.most_common(top_n_patterns): + # Backticks + escape any stray backticks in the pattern itself. + safe = pattern.replace("`", "ʼ") + fh.write(f"- `{safe}` — {count}\n") + fh.write("\n") + + # --- Per-version index table. --- + fh.write("| Dandiset | Version | Problems | Status | Modified |\n") + fh.write("|---|---|---:|---|---|\n") + for r in sorted(records, key=lambda x: (x["dandiset_id"], x["version"])): + problems_cell = ( + f"[{r['problem_count']}]({r['_summary_link']})" + if r["problem_count"] + else f"[OK]({r['_summary_link']})" + ) + # ``status`` and ``modified`` come from each version's ``info.json``; + # ``_attach_info`` has already stashed them onto the record so we + # can render the table without touching the filesystem here. + status = r.get("_status", "?") + modified = r.get("_modified", "?") + fh.write( + f"| {r['dandiset_id']} | {r['version']} | {problems_cell} " + f"| {status} | {modified} |\n" + ) + fh.write("\n") + + +def _attach_info(records: list[dict], data_dir: Path) -> None: + """Stitch the matching ``info.json`` fields onto each record. + + We do this once after loading so ``_render_bucket`` can render the + per-version table without re-reading the filesystem in a loop. + """ + for r in records: + info_path = data_dir / r["dandiset_id"] / r["version"] / "info.json" + try: + info = json.loads(info_path.read_text()) + r["_status"] = info.get("status") + r["_modified"] = info.get("modified") + except (FileNotFoundError, json.JSONDecodeError): + pass + + +def _render_report( + out_path: Path, + records: list[dict], + *, + commit_hash: str, + commit_date: str, + branch: str, + schema: str, + top_n_patterns: int, +) -> None: + """Write the top-level ``REPORT.md`` based on ``records``.""" + # Group records into the (class, schemaVersion) buckets the report + # is organised around. + buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) + for r in records: + buckets[_bucket_key(r)].append(r) + + with out_path.open("w") as fh: + fh.write("# DANDI metadata — LinkML validation report\n\n") + fh.write(f"- **Branch:** `{branch}`\n") + fh.write(f"- **Commit:** `{commit_hash}`\n") + fh.write(f"- **Commit date:** {commit_date}\n") + fh.write(f"- **Schema:** `{schema}`\n") + fh.write(f"- **Total dandiset versions checked:** {len(records)}\n\n") + + n_valid = sum(1 for r in records if r["problem_count"] == 0) + fh.write( + f"**Overall:** {n_valid} valid / " + f"{len(records) - n_valid} with problems " + f"out of {len(records)} versions.\n\n" + ) + + # Draft section first (target class: Dandiset), then published. + for cls, heading in [ + ("Dandiset", "Draft versions (target class: `Dandiset`)"), + ( + "PublishedDandiset", + "Published versions (target class: `PublishedDandiset`)", + ), + ]: + cls_records = [r for r in records if r["target_class"] == cls] + if not cls_records: + continue + fh.write(f"## {heading}\n\n") + fh.write(f"Total: {len(cls_records)} versions.\n\n") + + # Sub-buckets: stable order — known schema versions first + # (descending so newest tends to appear first), unknowns last. + schema_versions = sorted( + {sv for c, sv in buckets if c == cls and sv != ""}, + reverse=True, + ) + if any(sv == "" for c, sv in buckets if c == cls): + schema_versions.append("") + + for sv in schema_versions: + bucket = buckets[(cls, sv)] + _render_bucket( + fh, + f"schemaVersion {sv}", + bucket, + top_n_patterns=top_n_patterns, + ) + + +# --------------------------------------------------------------------------- +# Typer entry point +# --------------------------------------------------------------------------- + + +@app.command() +def main( + report_root: Path = typer.Argument( + ..., + help="Top-level report directory, i.e. " + "linkml-validation-reports//. " + "Must contain a `data/` subdirectory of validated versions.", + ), + commit_hash: str = typer.Option( + ..., + "--commit-hash", + help="Full commit hash of the linkml-auto-converted tip " + "(included in the report header).", + ), + commit_date: str = typer.Option( + ..., + "--commit-date", + help="ISO-8601 commit date of the linkml-auto-converted tip.", + ), + branch: str = typer.Option( + "linkml-auto-converted", + "--branch", + help="Branch name to print in the report header.", + ), + schema: str = typer.Option( + "dandischema/models.yaml", + "--schema", + help="Schema path to print in the report header.", + ), + top_n_patterns: int = typer.Option( + 10, + "--top-n-patterns", + help="How many most-common problem patterns to list per bucket.", + ), + log_level: str = typer.Option("INFO", "--log-level", "-l"), +) -> None: + """Aggregate per-version validation outputs into a top-level REPORT.md.""" + logging.basicConfig( + format="[%(asctime)s] %(levelname)s %(name)s: %(message)s", + level=getattr(logging, log_level.upper()), + ) + + data_dir = report_root / "data" + if not data_dir.is_dir(): + raise typer.BadParameter(f"no data/ directory under {report_root}") + + records = _load_records(data_dir) + _attach_info(records, data_dir) + logger.info("loaded %d validation records", len(records)) + + out_path = report_root / "REPORT.md" + _render_report( + out_path, + records, + commit_hash=commit_hash, + commit_date=commit_date, + branch=branch, + schema=schema, + top_n_patterns=top_n_patterns, + ) + logger.info("wrote %s", out_path) + + +if __name__ == "__main__": + app() diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py new file mode 100644 index 00000000..3a9df468 --- /dev/null +++ b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Validate downloaded dandiset metadata using the ``linkml-validate`` CLI. + +This script is a thin orchestrator. It does **not** import the LinkML +validator as a Python library; instead, for every dandiset version +directory produced by ``fetch_metadata.py`` it shells out to:: + + linkml-validate -s -C /metadata.json + +where the target class is decided from the version's ``info.json``: + + * ``Dandiset`` — for the ``draft`` version + * ``PublishedDandiset`` — for any non-``draft`` (i.e. published) version + +For each version directory the script writes three sibling files: + + ``validation.txt`` — verbatim stdout / stderr of ``linkml-validate``, + i.e. the human-readable transcript you would + have seen if you'd run the CLI by hand. + + ``validation.json`` — small machine-readable wrapper containing the + exit code, the chosen target class, the number + of problem lines, and the parsed problem lines + themselves. The report generator reads this + file rather than re-parsing ``validation.txt``. + + ``SUMMARY.md`` — short markdown summary of the version's + validation outcome (linked from the top-level + report). + +Re-running is safe: by default versions that already have a +``validation.json`` are skipped. Pass ``--refresh`` to re-validate +everything. + +Example +------- +:: + + python validate_metadata.py linkml-validation-reports//data \\ + --schema dandischema/models.yaml +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +import re +import subprocess + +import typer + +logger = logging.getLogger("validate_metadata") + +app = typer.Typer(add_completion=False, help=__doc__.splitlines()[0]) + + +# --------------------------------------------------------------------------- +# Output parsing +# --------------------------------------------------------------------------- +# +# ``linkml-validate`` prints one validation problem per stdout line and +# exits non-zero if any problems were found. We classify each line as +# either a "problem line" or noise (blank lines, the "No issues found" +# banner some versions emit, etc.) and persist the problem lines into a +# JSON record so the report generator doesn't have to re-parse the text. + +_NOISE_PATTERNS = [ + re.compile(r"^\s*$"), # blank lines + re.compile(r"^No issues found", re.IGNORECASE), # success banner +] + + +def _is_problem_line(line: str) -> bool: + """Return True if ``line`` should be counted as a validation problem.""" + return not any(p.search(line) for p in _NOISE_PATTERNS) + + +def _run_linkml_validate( + schema: Path, target_class: str, metadata_file: Path +) -> subprocess.CompletedProcess[str]: + """Invoke ``linkml-validate`` and return the completed process. + + ``check=False`` because a non-zero exit code is the *expected* + outcome whenever the metadata fails validation — we want to record + that, not raise. + """ + cmd = [ + "linkml-validate", + "-s", + str(schema), + "-C", + target_class, + str(metadata_file), + ] + logger.debug("running %s", " ".join(cmd)) + return subprocess.run(cmd, check=False, capture_output=True, text=True) + + +# --------------------------------------------------------------------------- +# Per-directory validation +# --------------------------------------------------------------------------- + + +def _validate_one(schema: Path, version_dir: Path, *, refresh: bool) -> tuple[str, int]: + """Validate one ``/`` directory. + + Reads ``info.json`` (written by ``fetch_metadata.py``) to decide the + target class, runs ``linkml-validate`` on ``metadata.json``, then + writes ``validation.txt``, ``validation.json`` and ``SUMMARY.md`` + inside ``version_dir``. + + Returns ``(target_class, n_problem_lines)`` so the caller can log a + tally without re-reading the file. + """ + metadata_file = version_dir / "metadata.json" + info_file = version_dir / "info.json" + out_text = version_dir / "validation.txt" + out_json = version_dir / "validation.json" + out_md = version_dir / "SUMMARY.md" + + info = json.loads(info_file.read_text()) + is_published = bool(info.get("is_published")) + target_class = "PublishedDandiset" if is_published else "Dandiset" + + # Resume support: if we already have a JSON record for this version + # and the caller didn't pass --refresh, leave the directory alone. + if out_json.exists() and not refresh: + try: + existing = json.loads(out_json.read_text()) + return target_class, int(existing.get("problem_count", 0)) + except (json.JSONDecodeError, ValueError): + logger.warning( + "re-validating %s — existing validation.json is unreadable", + version_dir, + ) + + proc = _run_linkml_validate(schema, target_class, metadata_file) + + # --- Persist the raw transcript exactly as the CLI would print it. --- + transcript = proc.stdout + if proc.stderr: + transcript += "\n--- stderr ---\n" + proc.stderr + if not transcript.endswith("\n"): + transcript += "\n" + out_text.write_text(transcript) + + # --- Parse the transcript into a small JSON record. --- + problem_lines = [ + line.rstrip() for line in proc.stdout.splitlines() if _is_problem_line(line) + ] + record = { + "dandiset_id": info["dandiset_id"], + "version": info["version"], + "is_published": is_published, + "target_class": target_class, + "schema_version": info.get("schema_version"), + "exit_code": proc.returncode, + "problem_count": len(problem_lines), + "problems": problem_lines, + } + out_json.write_text(json.dumps(record, indent=2) + "\n") + + # --- Per-version markdown summary, linked from the top-level report. --- + md_lines = [ + f"# Validation summary — {info['dandiset_id']} @ {info['version']}", + "", + f"- **Target class:** `{target_class}`", + f"- **Status:** {info.get('status')}", + f"- **Modified:** {info.get('modified')}", + f"- **schemaVersion:** {info.get('schema_version')}", + f"- **`linkml-validate` exit code:** {proc.returncode}", + f"- **# problem lines:** {len(problem_lines)}", + "", + "## Files", + "", + "- [`metadata.json`](metadata.json) — raw metadata as fetched from the archive", + "- [`validation.txt`](validation.txt) — verbatim `linkml-validate` output", + "- [`validation.json`](validation.json) — parsed validation record", + "", + ] + if problem_lines: + md_lines += [ + "## First 20 problem lines", + "", + "```", + *problem_lines[:20], + "```", + ] + if len(problem_lines) > 20: + md_lines.append( + f"_… {len(problem_lines) - 20} more — see " + "[`validation.txt`](validation.txt)._" + ) + out_md.write_text("\n".join(md_lines) + "\n") + + return target_class, len(problem_lines) + + +# --------------------------------------------------------------------------- +# Typer entry point +# --------------------------------------------------------------------------- + + +@app.command() +def main( + root: Path = typer.Argument( + ..., + help="Top-level directory produced by fetch_metadata.py " + "(contains //metadata.json files).", + ), + schema: Path = typer.Option( + ..., + "--schema", + help="Path to dandischema/models.yaml (the LinkML schema).", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Re-validate even when validation.json already exists.", + ), + log_level: str = typer.Option("INFO", "--log-level", "-l"), +) -> None: + """Validate every ``//metadata.json`` under ``root``.""" + logging.basicConfig( + format="[%(asctime)s] %(levelname)s %(name)s: %(message)s", + level=getattr(logging, log_level.upper()), + ) + + version_dirs = sorted( + p for p in root.glob("*/*") if p.is_dir() and (p / "metadata.json").is_file() + ) + logger.info("found %d version directories to validate", len(version_dirs)) + + for vd in version_dirs: + try: + target, n = _validate_one(schema, vd, refresh=refresh) + logger.info( + "validated %s/%s as %s (%d problem lines)", + vd.parent.name, + vd.name, + target, + n, + ) + except Exception as e: + # Never let one broken dandiset abort the whole run. + logger.exception("validation failed for %s: %s", vd, e) + + +if __name__ == "__main__": + app() diff --git a/pyproject.toml b/pyproject.toml index 1697fd8f..ac6cbfb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ python = "3.10" [tool.hatch.envs.linkml-auto-converted] dependencies = [ "black", # This allows `gen-pydantic` to format the generated Pydantic models with Black formatting + "dandi", # DANDI client, used by scripts that fetch dandiset metadata from the archive "linkml==1.10.0", "linkml-runtime==1.10.0", "pydantic2linkml @ git+https://github.com/dandi/pydantic2linkml.git", From 4c21a627e7155c3b76a95c7c30ef33da674537af Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 27 Apr 2026 00:25:18 -0700 Subject: [PATCH 46/90] Make fetch_metadata.py write metadata.json + info.json all-or-nothing Previously a partial run could leave a corrupted metadata.json or info.json behind, and a subsequent run with the default refresh=False would happily skip the version because both files looked present. Now the function does all network work first, pre-renders both JSON payloads, and then writes them via temp files plus os.replace() so the destination paths only appear once both have been written successfully -- even under SIGKILL or other abrupt termination. Drive-by simplifications addressed inline review notes: - drop the parent.mkdir() side effect from the JSON writer; the caller now creates version_dir explicitly, - drop json.dumps(default=str) since we already isoformat() datetimes before stashing them into info, - use VersionStatus.value directly and skip the modified-is-None branch (per the dandi client's Version model both fields are non-optional), - use PEP 604 `int | None` for the --limit option type. --- .../scripts/fetch_metadata.py | 67 +++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py index 814ba02c..3d558653 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py @@ -34,8 +34,8 @@ import json import logging +import os from pathlib import Path -from typing import Optional from dandi.dandiapi import DandiAPIClient, RemoteDandiset import typer @@ -50,15 +50,15 @@ # --------------------------------------------------------------------------- -def _write_json(path: Path, data: object) -> None: - """Write ``data`` to ``path`` as pretty-printed JSON, creating parent - directories as needed. +def _dump_json(data: object) -> str: + """Serialize ``data`` as pretty-printed JSON with a trailing newline. - ``default=str`` lets us serialise ``datetime`` objects without having - to convert them by hand. + ``data`` is expected to be a structure of plain JSON-compatible types + (dicts, lists, strings, numbers, bools, ``None``). Datetime objects + must be converted by the caller — see how ``_fetch_version`` calls + ``.isoformat()`` before stashing values into ``info``. """ - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, indent=2, default=str) + "\n") + return json.dumps(data, indent=2) + "\n" def _fetch_version( @@ -71,6 +71,17 @@ def _fetch_version( ) -> None: """Fetch ``metadata.json`` and ``info.json`` for one version. + All-or-nothing on the destination paths: this function performs the + network calls first, then writes both files to temporary paths in + ``version_dir`` and only renames them into place once both have been + written successfully. If anything fails — a raised exception or even + abrupt termination of the process — neither destination file ever + appears in a partially-written state, so the resume guard at the top + of this function can trust ``metadata.json``/``info.json`` existence + as a signal that the version was previously fetched in full. (The + leftover ``.tmp`` files are harmless cruft that the next successful + fetch overwrites.) + Parameters ---------- dandiset: @@ -94,6 +105,7 @@ def _fetch_version( logger.debug("skip %s/%s (already downloaded)", dandiset.identifier, version_id) return + # --- Network: gather everything before touching the filesystem. --- # ``for_version`` returns a fresh handle bound to the requested version, # which is what ``get_raw_metadata`` and ``get_version`` need to operate on. ds_at_version = dandiset.for_version(version_id) @@ -104,20 +116,37 @@ def _fetch_version( "dandiset_id": dandiset.identifier, "version": version_id, "is_published": is_published, - # ``status`` is an enum-like object on the client; ``.value`` if - # available, otherwise fall back to ``str()``. - "status": getattr(version_info.status, "value", str(version_info.status)), - "modified": ( - version_info.modified.isoformat() - if version_info.modified is not None - else None - ), + # ``status`` is a ``VersionStatus`` enum member on the client. + "status": version_info.status.value, + # ``modified`` is a non-optional ``datetime`` per the ``Version`` + # model, so an ``isoformat()`` is always safe. + "modified": version_info.modified.isoformat(), # The raw metadata's ``schemaVersion`` field is the dimension we # want to group by in the top-level report, so capture it now. "schema_version": raw.get("schemaVersion"), } - _write_json(metadata_file, raw) - _write_json(info_file, info) + + # --- Filesystem: write to .tmp paths then rename into place. --- + # Pre-rendering the JSON before opening any file keeps any + # serialization error from leaving stray ``.tmp`` files behind. + metadata_text = _dump_json(raw) + info_text = _dump_json(info) + + version_dir.mkdir(parents=True, exist_ok=True) + metadata_tmp = metadata_file.with_suffix(metadata_file.suffix + ".tmp") + info_tmp = info_file.with_suffix(info_file.suffix + ".tmp") + metadata_tmp.write_text(metadata_text) + info_tmp.write_text(info_text) + # ``os.replace`` performs the POSIX ``rename(2)`` syscall, which the + # kernel cannot leave half-finished: either the destination ends up + # pointing at the new content, or it stays as it was before the call + # (i.e. nonexistent on the first fetch). That guarantee is what + # makes the all-or-nothing behavior above hold even under SIGKILL, + # since simply ``write_text``-ing the final paths would leave a + # truncated file behind if the process were killed mid-write. + os.replace(metadata_tmp, metadata_file) + os.replace(info_tmp, info_file) + logger.info("fetched %s/%s", dandiset.identifier, version_id) @@ -145,7 +174,7 @@ def main( "--refresh", help="Re-download versions whose metadata is already on disk.", ), - limit: Optional[int] = typer.Option( + limit: int | None = typer.Option( None, "--limit", help="Process at most N dandisets (useful for smoke tests).", From 896d4fcbeaa17a693f312640000db57e81272493 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 27 Apr 2026 00:25:19 -0700 Subject: [PATCH 47/90] Use American English spelling in skill scripts Renames _normalise_problem -> _normalize_problem (and updates the caller), and replaces 'organised' with 'organized' in a comment. --- .../scripts/generate_report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py index 4e0c4631..fbc45c46 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -77,7 +77,7 @@ _SOURCE_PREFIX_RE = re.compile(r"^(\[[A-Z]+\])\s+\[[^\]]+\]\s*") -def _normalise_problem(line: str) -> str: +def _normalize_problem(line: str) -> str: """Drop the per-file source prefix from a problem line. Turns ``"[ERROR] [.../000003/.../metadata.json/0] foo in /bar"`` into @@ -153,7 +153,7 @@ def _render_bucket( pattern_counter: Counter[str] = Counter() for r in records: for problem in r.get("problems", []): - pattern_counter[_normalise_problem(problem)] += 1 + pattern_counter[_normalize_problem(problem)] += 1 if pattern_counter: fh.write(f"**Top {top_n_patterns} problem patterns:**\n\n") for pattern, count in pattern_counter.most_common(top_n_patterns): @@ -211,7 +211,7 @@ def _render_report( ) -> None: """Write the top-level ``REPORT.md`` based on ``records``.""" # Group records into the (class, schemaVersion) buckets the report - # is organised around. + # is organized around. buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) for r in records: buckets[_bucket_key(r)].append(r) From b2270b9bb69851f8630076ad249a9cb5b95af470 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 28 Apr 2026 22:29:20 -0700 Subject: [PATCH 48/90] Use LinkML Python API for validation, with structured JSON output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `validate_metadata.py` now drives validation through the LinkML `Validator` Python API (configured with the same `JsonschemaValidationPlugin(closed=True)` the CLI uses by default) instead of shelling out. One run produces both a structured `validation.json` (carrying the `ValidationResult` fields plus `source.validator` / `validator_value` for grouping) and a `validation.txt` transcript byte-equivalent to what `linkml-validate` would have printed. `@context` is stripped from the data instance before validation so the JSON-schema check doesn't flag the JSON-LD framing key as an unexpected property — see linkml/linkml#3442. `generate_report.py` is updated in lockstep: problem patterns are now grouped by `[severity] message`, sourced from the structured record rather than regex-scrubbed CLI text. Co-Authored-By: Claude Opus 4.7 --- .../scripts/generate_report.py | 35 +-- .../scripts/validate_metadata.py | 227 ++++++++++-------- 2 files changed, 147 insertions(+), 115 deletions(-) diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py index fbc45c46..d22eb0a9 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -53,7 +53,6 @@ import json import logging from pathlib import Path -import re import typer @@ -67,23 +66,25 @@ # --------------------------------------------------------------------------- -# A line emitted by ``linkml-validate`` looks like:: -# -# [ERROR] [path/to/metadata.json/0] in /json/pointer -# -# The bracketed source-file segment is per-version noise that prevents -# similar errors across dandisets from grouping together. This regex -# strips it so we can count error patterns meaningfully. -_SOURCE_PREFIX_RE = re.compile(r"^(\[[A-Z]+\])\s+\[[^\]]+\]\s*") +def _problem_pattern(problem: dict) -> str: + """Build a grouping key for one structured problem record. - -def _normalize_problem(line: str) -> str: - """Drop the per-file source prefix from a problem line. - - Turns ``"[ERROR] [.../000003/.../metadata.json/0] foo in /bar"`` into - ``"[ERROR] foo in /bar"``, which groups across dandisets. + ``validate_metadata.py`` writes each problem as a dict with at + least ``severity`` and ``message`` (and, for JSON-schema-backed + validation, a ``source.validator`` keyword). The path-prefixed + text the CLI prints carries no information not already in these + fields, so we group on ``[severity] message`` and prepend the + failing JSON-schema validator keyword when available — that lets + similar errors group across dandisets without regex scrubbing. """ - return _SOURCE_PREFIX_RE.sub(r"\1 ", line, count=1) + severity = problem.get("severity", "?") + message = problem.get("message", "") + src = problem.get("source") or {} + validator = src.get("validator") + prefix = f"[{severity}]" + if validator: + prefix += f" <{validator}>" + return f"{prefix} {message}" def _load_records(data_dir: Path) -> list[dict]: @@ -153,7 +154,7 @@ def _render_bucket( pattern_counter: Counter[str] = Counter() for r in records: for problem in r.get("problems", []): - pattern_counter[_normalize_problem(problem)] += 1 + pattern_counter[_problem_pattern(problem)] += 1 if pattern_counter: fh.write(f"**Top {top_n_patterns} problem patterns:**\n\n") for pattern, count in pattern_counter.most_common(top_n_patterns): diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py index 3a9df468..bbc8f282 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py @@ -1,33 +1,32 @@ #!/usr/bin/env python3 -"""Validate downloaded dandiset metadata using the ``linkml-validate`` CLI. - -This script is a thin orchestrator. It does **not** import the LinkML -validator as a Python library; instead, for every dandiset version -directory produced by ``fetch_metadata.py`` it shells out to:: - - linkml-validate -s -C /metadata.json +"""Validate downloaded dandiset metadata using the LinkML validator Python API. + +For every dandiset version directory produced by ``fetch_metadata.py`` this +script runs LinkML validation against ``dandischema/models.yaml`` and emits +three sibling files in the version directory: + + ``validation.json`` — machine-readable record. Wraps the structured + ``ValidationResult`` objects from the validator + (severity, message, instance index, JSON-pointer + path, validator name, etc.) plus a small header + identifying the dandiset/version/target class. + + ``validation.txt`` — human-readable transcript that is byte-equivalent + to what ``linkml-validate`` would have printed for + the same inputs (same ``[severity] [source/idx] + message`` template, same ``No issues found`` + banner, same exit-code semantics). One run, two + outputs — we don't shell out to the CLI. + + ``SUMMARY.md`` — short markdown summary of the version's + validation outcome (linked from the top-level + report). -where the target class is decided from the version's ``info.json``: +The target class is decided from the version's ``info.json``: * ``Dandiset`` — for the ``draft`` version * ``PublishedDandiset`` — for any non-``draft`` (i.e. published) version -For each version directory the script writes three sibling files: - - ``validation.txt`` — verbatim stdout / stderr of ``linkml-validate``, - i.e. the human-readable transcript you would - have seen if you'd run the CLI by hand. - - ``validation.json`` — small machine-readable wrapper containing the - exit code, the chosen target class, the number - of problem lines, and the parsed problem lines - themselves. The report generator reads this - file rather than re-parsing ``validation.txt``. - - ``SUMMARY.md`` — short markdown summary of the version's - validation outcome (linked from the top-level - report). - Re-running is safe: by default versions that already have a ``validation.json`` are skipped. Pass ``--refresh`` to re-validate everything. @@ -45,9 +44,10 @@ import json import logging from pathlib import Path -import re -import subprocess +from linkml.validator import Validator +from linkml.validator.plugins import JsonschemaValidationPlugin +from linkml.validator.report import Severity, ValidationResult import typer logger = logging.getLogger("validate_metadata") @@ -56,45 +56,46 @@ # --------------------------------------------------------------------------- -# Output parsing +# Result rendering # --------------------------------------------------------------------------- -# -# ``linkml-validate`` prints one validation problem per stdout line and -# exits non-zero if any problems were found. We classify each line as -# either a "problem line" or noise (blank lines, the "No issues found" -# banner some versions emit, etc.) and persist the problem lines into a -# JSON record so the report generator doesn't have to re-parse the text. - -_NOISE_PATTERNS = [ - re.compile(r"^\s*$"), # blank lines - re.compile(r"^No issues found", re.IGNORECASE), # success banner -] - - -def _is_problem_line(line: str) -> bool: - """Return True if ``line`` should be counted as a validation problem.""" - return not any(p.search(line) for p in _NOISE_PATTERNS) -def _run_linkml_validate( - schema: Path, target_class: str, metadata_file: Path -) -> subprocess.CompletedProcess[str]: - """Invoke ``linkml-validate`` and return the completed process. +def _result_to_dict(r: ValidationResult) -> dict: + """Serialize one ``ValidationResult`` into a JSON-friendly dict. - ``check=False`` because a non-zero exit code is the *expected* - outcome whenever the metadata fails validation — we want to record - that, not raise. + Pydantic excludes the ``source`` field from default serialization + (it's an arbitrary plugin-defined object), but for JSON-schema-based + validation it carries useful grouping signals — the failing + validator keyword (e.g. ``"required"``, ``"enum"``) and the value + that triggered the failure. We pull those out by hand so the + downstream report can group by validator without re-parsing + messages. """ - cmd = [ - "linkml-validate", - "-s", - str(schema), - "-C", - target_class, - str(metadata_file), - ] - logger.debug("running %s", " ".join(cmd)) - return subprocess.run(cmd, check=False, capture_output=True, text=True) + # ``instance`` echoes the full data instance back into every result, + # which would duplicate ``metadata.json`` per-problem and bloat the + # record without adding any information the consumer doesn't already + # have. Drop it. + d = r.model_dump(mode="json", exclude={"instance"}) + src = r.source + if src is not None: + d["source"] = { + "validator": getattr(src, "validator", None), + "validator_value": getattr(src, "validator_value", None), + } + return d + + +def _format_cli_line(r: ValidationResult, source_label: str) -> str: + """Format one result the way ``linkml-validate`` prints it. + + Mirrors the f-string in ``linkml/validator/cli.py`` exactly so the + transcript stays byte-equivalent to the CLI's stdout. + """ + # Match the CLI's f-string interpolation exactly: no fallback. If + # ``instance_index`` is ``None`` (e.g. a result emitted for a non-list + # instance), the literal ``"None"`` is what the CLI would print, and + # mirroring that keeps the transcript byte-equivalent. + return f"[{r.severity.value}] [{source_label}/{r.instance_index}] {r.message}" # --------------------------------------------------------------------------- @@ -102,16 +103,18 @@ def _run_linkml_validate( # --------------------------------------------------------------------------- -def _validate_one(schema: Path, version_dir: Path, *, refresh: bool) -> tuple[str, int]: +def _validate_one( + validator: Validator, version_dir: Path, *, refresh: bool +) -> tuple[str, int]: """Validate one ``/`` directory. Reads ``info.json`` (written by ``fetch_metadata.py``) to decide the - target class, runs ``linkml-validate`` on ``metadata.json``, then - writes ``validation.txt``, ``validation.json`` and ``SUMMARY.md`` - inside ``version_dir``. + target class, runs the validator on ``metadata.json``, and writes + ``validation.json`` / ``validation.txt`` / ``SUMMARY.md`` alongside + it. - Returns ``(target_class, n_problem_lines)`` so the caller can log a - tally without re-reading the file. + Returns ``(target_class, n_results)`` so the caller can log a tally + without re-reading the file. """ metadata_file = version_dir / "metadata.json" info_file = version_dir / "info.json" @@ -135,29 +138,46 @@ def _validate_one(schema: Path, version_dir: Path, *, refresh: bool) -> tuple[st version_dir, ) - proc = _run_linkml_validate(schema, target_class, metadata_file) - - # --- Persist the raw transcript exactly as the CLI would print it. --- - transcript = proc.stdout - if proc.stderr: - transcript += "\n--- stderr ---\n" + proc.stderr - if not transcript.endswith("\n"): - transcript += "\n" - out_text.write_text(transcript) - - # --- Parse the transcript into a small JSON record. --- - problem_lines = [ - line.rstrip() for line in proc.stdout.splitlines() if _is_problem_line(line) - ] + # --- Run validation once. --- + raw = json.loads(metadata_file.read_text()) + # ``@context`` is a JSON-LD framing field that's not part of the + # ``Dandiset`` / ``PublishedDandiset`` LinkML class definitions, so a + # closed-world JSON-schema check flags it as an unexpected property + # (see linkml/linkml#3442). Strip it before validation so we don't + # drown the report in noise that has nothing to do with the model. + raw.pop("@context", None) + report = validator.validate(raw, target_class=target_class) + results: list[ValidationResult] = report.results + + # ``linkml-validate``'s exit code is 1 iff any ERROR-severity result + # is present, else 0. We replicate that for downstream consumers + # that key off ``exit_code``. + has_error = any(r.severity is Severity.ERROR for r in results) + exit_code = 1 if has_error else 0 + + # --- Render the human-readable transcript (CLI-equivalent). --- + # The CLI prints ``loader.source`` as the bracketed path; for a + # file-backed JsonLoader that's the file path string, so use the + # same here. + source_label = str(metadata_file) + if results: + transcript_lines = [_format_cli_line(r, source_label) for r in results] + else: + # Mirrors the CLI's success banner so byte-equivalence holds in + # the zero-results case too. + transcript_lines = ["No issues found"] + out_text.write_text("\n".join(transcript_lines) + "\n") + + # --- Persist the structured record. --- record = { "dandiset_id": info["dandiset_id"], "version": info["version"], "is_published": is_published, "target_class": target_class, "schema_version": info.get("schema_version"), - "exit_code": proc.returncode, - "problem_count": len(problem_lines), - "problems": problem_lines, + "exit_code": exit_code, + "problem_count": len(results), + "problems": [_result_to_dict(r) for r in results], } out_json.write_text(json.dumps(record, indent=2) + "\n") @@ -169,32 +189,32 @@ def _validate_one(schema: Path, version_dir: Path, *, refresh: bool) -> tuple[st f"- **Status:** {info.get('status')}", f"- **Modified:** {info.get('modified')}", f"- **schemaVersion:** {info.get('schema_version')}", - f"- **`linkml-validate` exit code:** {proc.returncode}", - f"- **# problem lines:** {len(problem_lines)}", + f"- **Equivalent `linkml-validate` exit code:** {exit_code}", + f"- **# problems:** {len(results)}", "", "## Files", "", "- [`metadata.json`](metadata.json) — raw metadata as fetched from the archive", - "- [`validation.txt`](validation.txt) — verbatim `linkml-validate` output", - "- [`validation.json`](validation.json) — parsed validation record", + "- [`validation.txt`](validation.txt) — `linkml-validate`-equivalent transcript", + "- [`validation.json`](validation.json) — structured validation record", "", ] - if problem_lines: + if results: md_lines += [ - "## First 20 problem lines", + "## First 20 problems", "", "```", - *problem_lines[:20], + *transcript_lines[:20], "```", ] - if len(problem_lines) > 20: + if len(results) > 20: md_lines.append( - f"_… {len(problem_lines) - 20} more — see " + f"_… {len(results) - 20} more — see " "[`validation.txt`](validation.txt)._" ) out_md.write_text("\n".join(md_lines) + "\n") - return target_class, len(problem_lines) + return target_class, len(results) # --------------------------------------------------------------------------- @@ -227,6 +247,16 @@ def main( level=getattr(logging, log_level.upper()), ) + # Build one ``Validator`` and reuse it across every version: parsing + # the schema is the expensive part, and the plugin configuration + # below matches the ``linkml-validate`` CLI default + # (``JsonschemaValidationPlugin`` with ``closed=True``), so the + # results we collect are the same ones the CLI would have emitted. + validator = Validator( + schema, + validation_plugins=[JsonschemaValidationPlugin(closed=True)], + ) + version_dirs = sorted( p for p in root.glob("*/*") if p.is_dir() and (p / "metadata.json").is_file() ) @@ -234,17 +264,18 @@ def main( for vd in version_dirs: try: - target, n = _validate_one(schema, vd, refresh=refresh) + target, n = _validate_one(validator, vd, refresh=refresh) + except Exception as e: + # Never let one broken dandiset abort the whole run. + logger.exception("validation failed for %s: %s", vd, e) + else: logger.info( - "validated %s/%s as %s (%d problem lines)", + "validated %s/%s as %s (%d problems)", vd.parent.name, vd.name, target, n, ) - except Exception as e: - # Never let one broken dandiset abort the whole run. - logger.exception("validation failed for %s: %s", vd, e) if __name__ == "__main__": From 6b04389bf21359a05d2ca3235477d47c88debc32 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 11:21:21 -0700 Subject: [PATCH 49/90] Rename "Status" column to "API Status" in REPORT.md The column carries the per-version `status` field returned by the DANDI Archive API (e.g. `VALID`, `Published`), not anything the LinkML validator computed. Labelling it "API Status" makes that distinction explicit so a reader doesn't mistake it for a validation outcome. Co-Authored-By: Claude Opus 4.7 --- .../dandi-linkml-validation-report/scripts/generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py index d22eb0a9..b657ca96 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -164,7 +164,7 @@ def _render_bucket( fh.write("\n") # --- Per-version index table. --- - fh.write("| Dandiset | Version | Problems | Status | Modified |\n") + fh.write("| Dandiset | Version | Problems | API Status | Modified |\n") fh.write("|---|---|---:|---|---|\n") for r in sorted(records, key=lambda x: (x["dandiset_id"], x["version"])): problems_cell = ( From f5f913775294ef0688867b994a25c32b6e19e94a Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 11:21:59 -0700 Subject: [PATCH 50/90] Add SKILL.md and reference docs for dandi-linkml-validation-report Authors the skill following the agentskills.io specification: - SKILL.md carries the required `name` / `description` frontmatter plus a `compatibility` note for the hatch env, and a tight workflow body (when to use, prerequisites, three-stage pipeline with exact `hatch run` commands). - references/OUTPUT.md documents the on-disk layout and JSON field shapes the pipeline emits, so callers don't have to read the scripts to understand the output. - references/DESIGN.md captures the rationale behind non-obvious choices (LinkML Python API instead of CLI, closed=True plugin, `@context` strip per linkml/linkml#3442, byte-equivalent CLI transcript, all-or-nothing fetch writes, resume semantics). Split this way to take advantage of progressive disclosure: SKILL.md stays under ~100 lines for the activation tier, with the deeper material loaded only when the agent actually needs it. Co-Authored-By: Claude Opus 4.7 --- .../dandi-linkml-validation-report/SKILL.md | 94 ++++++++++ .../references/DESIGN.md | 136 +++++++++++++++ .../references/OUTPUT.md | 165 ++++++++++++++++++ 3 files changed, 395 insertions(+) create mode 100644 .claude/skills/dandi-linkml-validation-report/SKILL.md create mode 100644 .claude/skills/dandi-linkml-validation-report/references/DESIGN.md create mode 100644 .claude/skills/dandi-linkml-validation-report/references/OUTPUT.md diff --git a/.claude/skills/dandi-linkml-validation-report/SKILL.md b/.claude/skills/dandi-linkml-validation-report/SKILL.md new file mode 100644 index 00000000..cb42aef4 --- /dev/null +++ b/.claude/skills/dandi-linkml-validation-report/SKILL.md @@ -0,0 +1,94 @@ +--- +name: dandi-linkml-validation-report +description: Generate a Markdown report assessing how `dandischema/models.yaml` (the LinkML schema) validates against real DANDI Archive Dandiset metadata. Use when the user wants to assess schema fitness across the archive, investigate a class of validation failure across many dandisets, or compare before/after for a schema change. Covers fetching raw metadata for every dandiset (draft + every published version), running closed-world JSON-schema validation via the LinkML Python API, and aggregating per-version results into a top-level REPORT.md bucketed by target class (Dandiset / PublishedDandiset) × schemaVersion. +compatibility: Requires the `linkml-auto-converted` hatch env defined in this repo's pyproject.toml (provides linkml, linkml-runtime, dandi, typer) and network access to a DANDI Archive instance. +--- + +# DANDI LinkML validation report + +Three-stage pipeline that fetches Dandiset metadata, validates it against +`dandischema/models.yaml`, and aggregates the results into a Markdown +report. Each stage is a Typer-based script under `scripts/`; each is +idempotent and resumable. + +## When to use + +- After updating the LinkML schema (or its Pydantic source in + `dandischema.models`) — see what breaks across the archive. +- To investigate the spread of a specific validation failure across + dandisets. +- To produce a before/after diff of schema changes. + +## Prerequisites + +- The `linkml-auto-converted` hatch env exists (defined in + `pyproject.toml`). +- `dandischema/models.yaml` is present and reflects the schema you want + to validate against. Typically, you stay on the `linkml-conversion` + branch and pull the YAML from the auto-generated branch: + + ```sh + git restore --source linkml-auto-converted -- dandischema/models.yaml + ``` + +- Network access to the target DANDI instance (default: dani). + +## Workflow + +Pick a short SHA of the schema commit to namespace the run: + +```sh +SHA=$(git rev-parse --short=7 linkml-auto-converted) +ROOT=linkml-validation-reports/$SHA +``` + +### 1. Fetch metadata + +```sh +hatch run linkml-auto-converted:python \ + .claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py \ + $ROOT/data +``` + +Downloads `metadata.json` + `info.json` for every dandiset's draft and +every published version into `$ROOT/data///`. +Re-running is safe — already-downloaded versions are skipped unless +`--refresh` is passed. `--limit N` truncates to N dandisets for smoke +tests. `-i ` selects a non-production DANDI instance. + +### 2. Validate + +```sh +hatch run linkml-auto-converted:python \ + .claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py \ + $ROOT/data --schema dandischema/models.yaml +``` + +For each version directory, runs LinkML's `Validator` Python API once +and writes `validation.json` (structured results), `validation.txt` +(byte-equivalent to `linkml-validate` CLI output), and `SUMMARY.md`. +Drafts are validated against `Dandiset`; published versions against +`PublishedDandiset`. `--refresh` re-validates already-validated +versions. + +### 3. Generate report + +```sh +hatch run linkml-auto-converted:python \ + .claude/skills/dandi-linkml-validation-report/scripts/generate_report.py \ + $ROOT \ + --commit-hash $(git rev-parse linkml-auto-converted) \ + --commit-date $(git show -s --format=%cI linkml-auto-converted) +``` + +Writes `$ROOT/REPORT.md`: overall counts, then per-bucket tables +(target class × schemaVersion) with top error patterns and links to +each version's `SUMMARY.md`. + +## Further reading + +- [references/OUTPUT.md](references/OUTPUT.md) — directory layout, + `info.json` / `validation.json` field shapes, REPORT.md structure. +- [references/DESIGN.md](references/DESIGN.md) — key design choices: + closed-world JSON-schema plugin, `@context` strip, byte-equivalent + CLI transcript, all-or-nothing fetch writes, resume semantics. diff --git a/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md b/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md new file mode 100644 index 00000000..3c08ee32 --- /dev/null +++ b/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md @@ -0,0 +1,136 @@ +# Design notes + +Rationale for non-obvious choices in the pipeline. Read this when you're +modifying a script and want to know why something was done a particular +way; the SKILL.md and OUTPUT.md cover the *what*. + +## LinkML Python API, not the CLI + +`validate_metadata.py` drives the `linkml.validator.Validator` class +directly rather than shelling out to `linkml-validate`. Reasons: + +- One validation run produces both the structured `validation.json` + and the human-readable `validation.txt`. Shelling out would force + either a second run or downstream regex parsing of CLI text. +- `linkml-validate` has no JSON output flag (verified against the + installed `linkml/validator/cli.py`), and its `--config` only + externalizes the same arguments — there is no formatter knob. +- Subprocess overhead is meaningful when iterating ~900 dandisets + with a few thousand version directories. + +`Validator` is constructed once and reused across every version +directory; schema parsing is the expensive part. + +## Closed-world JSON-schema plugin + +We pass `JsonschemaValidationPlugin(closed=True)` when constructing the +validator. That matches the CLI's default (`{"JsonschemaValidationPlugin": +{"closed": True}}` in `linkml/validator/cli.py`), so the results we +collect are the same ones `linkml-validate` would emit. + +`closed=True` causes LinkML to emit `additionalProperties: false` in +the generated JSON Schema — extra keys on a class become validation +errors. Switching to `closed=False` would silently accept unknown +fields and mask real bugs (e.g. `includeInCitation` on `Affiliation`), +so it's not an acceptable knob for "ignore one specific extra key". + +## `@context` is stripped before validation + +DANDI metadata carries a top-level JSON-LD framing key (`@context`) +that isn't part of the `Dandiset` / `PublishedDandiset` LinkML class +definitions. With `closed=True` it shows up as an +`additionalProperties` violation — see linkml/linkml#3442. + +We `raw.pop("@context", None)` on the loaded dict before passing it to +`validator.validate(...)`. Cheap, local to the validator step, and +doesn't touch the saved `metadata.json` (which we keep verbatim). +Considered alternatives: + +- Open up `closed=False` globally — too broad; loses signal. +- Post-validation filter — runs the validator, gets a known-noise + result, throws it away. Backwards from intent. +- Custom plugin subclass — tracking LinkML internals just to skip one + message; not worth it. +- Schema patch in `models.yaml` — `models.yaml` is auto-generated from + `dandischema.models`, so the change would have to live in the + Pydantic source. Too far away from the narrow concern. + +## CLI byte-equivalent transcript + +`validation.txt` is rendered by formatting each `ValidationResult` +with the same f-string `linkml-validate` uses +(`linkml/validator/cli.py`): + +``` +[] [/] +``` + +Plus a single-line `No issues found` when zero results, plus an +`exit_code` field in the JSON header that mirrors the CLI's exit code +(1 iff any `ERROR` severity is present, else 0). We interpolate +`r.instance_index` directly with no fallback so a `None` (which would +appear if LinkML stops emitting `0` for non-list instances) renders as +the literal `"None"` — same as the CLI. Today the JSON-schema plugin +always emits `0` for our single-instance `validate(raw, ...)` calls, +so the field renders as `0` in practice. + +## Structured problem records + +`validation.json` does *not* persist the CLI text and ask the report +to parse it. Each problem is the dict-form of a `ValidationResult` +plus the bits of `source` that matter for grouping +(`source.validator`, `source.validator_value`). + +`source` is excluded from default pydantic serialization (it's a +plugin-defined object), so `_result_to_dict` extracts those two +attributes by hand. The `instance` field — which echoes the full data +instance back into every result — is excluded to avoid duplicating +`metadata.json` per problem. + +`generate_report.py` groups patterns by `[] <> +`, drawing the validator keyword from the structured record. +That groups across dandisets without regex-scrubbing the per-file path +out of the CLI text, and surfaces the failing JSON-schema keyword +(e.g. `additionalProperties`, `enum`, `format`) directly in the +report. + +## All-or-nothing fetch writes + +`fetch_metadata.py` writes `metadata.json` and `info.json` together or +not at all. Mechanism: + +1. Network calls first — `get_raw_metadata()` and `get_version()` both + complete before any filesystem write. +2. Pre-render both JSON strings. +3. Write each to a `.tmp` sibling. +4. `os.replace(...)` each `.tmp` onto its final path. + +`os.replace` performs the POSIX `rename(2)` syscall; either the +destination ends up at the new content or it stays as it was before. +That guarantee is what makes the resume guard correct: the next run +checks for *both* `metadata.json` and `info.json` and skips only if +both exist, so a process killed mid-fetch never leaves a half-written +file that the resume guard would mistake for completion. (Leftover +`.tmp` files are harmless cruft that the next successful fetch +overwrites.) + +We considered a `try/except` around plain writes to the final paths. +That's strictly worse: a SIGKILL between writing `metadata.json` fully +and `info.json` partially leaves both files existing with `info.json` +truncated, and the resume guard would skip the retry. The temp+rename +pattern can't reach that state. + +## Resume semantics + +Each stage skips work it's already done: + +- `fetch_metadata.py`: skip a version when both `metadata.json` and + `info.json` exist, unless `--refresh`. +- `validate_metadata.py`: skip a version when `validation.json` + exists and parses, unless `--refresh`. If it parses but is + unreadable, log a warning and re-validate. +- `generate_report.py`: stateless; always rewrites `REPORT.md` from + whatever `validation.json` files are present. + +Failures in either of the first two stages are caught at the version +level — one broken dandiset doesn't abort the run. diff --git a/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md b/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md new file mode 100644 index 00000000..70fde856 --- /dev/null +++ b/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md @@ -0,0 +1,165 @@ +# Output anatomy + +Reference for the on-disk layout produced by the three pipeline stages +and the shape of every JSON artifact. The top-level `SKILL.md` describes +*how* to run the pipeline; this file describes *what it produces*. + +## Directory layout + +``` +linkml-validation-reports// +├── REPORT.md # written by generate_report.py +└── data/ + └── / # e.g. 000003 + ├── draft/ # draft version + │ ├── metadata.json # written by fetch_metadata.py + │ ├── info.json # written by fetch_metadata.py + │ ├── validation.json # written by validate_metadata.py + │ ├── validation.txt # written by validate_metadata.py + │ └── SUMMARY.md # written by validate_metadata.py + └── / # e.g. 0.230629.1955 + └── … # same five files +``` + +`` is whatever the caller of `generate_report.py` passed for +`--commit-hash` (typically `git rev-parse --short=7 linkml-auto-converted`). +The pipeline doesn't inspect or enforce it — it's just a namespace for +the run. + +## `metadata.json` + +Verbatim raw metadata as returned by `RemoteDandiset.get_raw_metadata()`. +Pretty-printed JSON. Not modified after the fetch — every downstream +consumer reads this as the authoritative archive payload. + +## `info.json` + +Side-car written alongside `metadata.json` so the validator and the +report can pick the right target class and group records without +re-querying the archive. + +```json +{ + "dandiset_id": "000003", + "version": "0.230629.1955", + "is_published": true, + "status": "VALID", + "modified": "2023-06-29T19:55:35.080882+00:00", + "schema_version": "0.6.4" +} +``` + +| Field | Source | +|------------------|------------------------------------------------------| +| `dandiset_id` | `RemoteDandiset.identifier` | +| `version` | Version id, or the literal string `"draft"` | +| `is_published` | `False` for the draft version, `True` otherwise | +| `status` | `version_info.status.value` (DANDI `VersionStatus`) | +| `modified` | `version_info.modified.isoformat()` (always present) | +| `schema_version` | `metadata["schemaVersion"]` (may be `null`) | + +`metadata.json` and `info.json` are written all-or-nothing: each is +written to a `.tmp` sibling first, then `os.replace`'d into place. If +the fetch is killed mid-write, neither final path appears, so the +resume guard treats that version as unfetched. + +## `validation.json` + +Structured validation record written by `validate_metadata.py`. One +record per version directory: + +```json +{ + "dandiset_id": "000003", + "version": "draft", + "is_published": false, + "target_class": "Dandiset", + "schema_version": "0.6.0", + "exit_code": 1, + "problem_count": 3, + "problems": [ { … }, … ] +} +``` + +Each entry in `problems` is a JSON-friendly serialization of one +LinkML `ValidationResult`: + +```json +{ + "type": "jsonschema validation", + "severity": "ERROR", + "message": "'Dandiset' is not one of ['PublishedDandiset'] in /schemaKey", + "instance_index": 0, + "instantiates": "PublishedDandiset", + "context": [], + "source": { + "validator": "enum", + "validator_value": ["PublishedDandiset"] + } +} +``` + +Notes on the fields: + +- `instance` (the full data instance the validator was given) is + excluded — it would duplicate `metadata.json` per problem and bloat + the file. Look at the sibling `metadata.json` if you need the + payload. +- `source.validator` / `source.validator_value` come from the + `JsonschemaValidationPlugin` and identify the failing JSON-schema + keyword (e.g. `additionalProperties`, `enum`, `format`, + `required`). The report generator groups patterns by this field so + similar failures across dandisets count together. +- `exit_code` is `1` if any `ERROR`-severity result is present, else + `0` — matches `linkml-validate`'s exit-code semantics. + +## `validation.txt` + +Byte-equivalent to what `linkml-validate -s -C +` would have printed. Each line: + +``` +[] [/] +``` + +When there are zero results, the file contains the single line +`No issues found` (matching the CLI's success banner). The literal +text comes from re-formatting `ValidationResult` objects — we don't +shell out to the CLI. + +## `SUMMARY.md` + +Per-version markdown summary linked from the top-level report. Header +lists target class, archive status, modified timestamp, raw +`schemaVersion`, equivalent CLI exit code, and problem count; body +inlines up to the first 20 problem lines and links back to the raw +files. + +## `REPORT.md` + +Top-level report written by `generate_report.py`. + +Header: + +- Branch name (default `linkml-auto-converted`) +- Commit hash + ISO commit date (passed in via flags) +- Schema path (default `dandischema/models.yaml`) +- Total versions and overall valid/with-problems count + +Then one section per target class: + +1. `Draft versions (target class: Dandiset)` +2. `Published versions (target class: PublishedDandiset)` + +Each section is sub-bucketed by `schemaVersion` (newest first; +`` last when present). Each sub-bucket contains: + +- A one-line headline (versions / valid / failing) +- Top-N grouped problem patterns of the form + `[] <> ` with counts +- A per-version table: `Dandiset | Version | Problems | API Status | + Modified`, where `Problems` links to each version's `SUMMARY.md`. + +The "API Status" column is the per-version `status` field that DANDI's +own API returns (`VALID`, `Published`, `Pending`, etc.) — not anything +the LinkML validator computed. From c30f141e17668454dd51db94da4fb082f1d935e4 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 12:13:57 -0700 Subject: [PATCH 51/90] Pre-approve git and hatch invocations in skill frontmatter Adds an `allowed-tools` declaration to SKILL.md so the three hatch-run pipeline commands and the supporting `git rev-parse`/`git show`/`git restore` calls don't prompt for permission while the skill is active. The patterns are scoped (`Bash(git:*) Bash(hatch:*)`), so unrelated shell commands still go through the normal approval flow. `allowed-tools` is marked experimental in the agentskills.io spec, so behavior may vary between agent implementations. Co-Authored-By: Claude Opus 4.7 --- .claude/skills/dandi-linkml-validation-report/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.claude/skills/dandi-linkml-validation-report/SKILL.md b/.claude/skills/dandi-linkml-validation-report/SKILL.md index cb42aef4..d2ed558b 100644 --- a/.claude/skills/dandi-linkml-validation-report/SKILL.md +++ b/.claude/skills/dandi-linkml-validation-report/SKILL.md @@ -2,6 +2,7 @@ name: dandi-linkml-validation-report description: Generate a Markdown report assessing how `dandischema/models.yaml` (the LinkML schema) validates against real DANDI Archive Dandiset metadata. Use when the user wants to assess schema fitness across the archive, investigate a class of validation failure across many dandisets, or compare before/after for a schema change. Covers fetching raw metadata for every dandiset (draft + every published version), running closed-world JSON-schema validation via the LinkML Python API, and aggregating per-version results into a top-level REPORT.md bucketed by target class (Dandiset / PublishedDandiset) × schemaVersion. compatibility: Requires the `linkml-auto-converted` hatch env defined in this repo's pyproject.toml (provides linkml, linkml-runtime, dandi, typer) and network access to a DANDI Archive instance. +allowed-tools: Bash(git:*) Bash(hatch:*) Read --- # DANDI LinkML validation report From 9bf74c53ab175b07ff3e308369c93f9120b81224 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 16:51:58 -0700 Subject: [PATCH 52/90] Rename top-level report from REPORT.md to README.md Naming the aggregated report `README.md` means GitHub renders it automatically as the landing view when the output directory is presented as a repo, so a reader lands on the validation report without having to click into a file. The on-disk layout is otherwise unchanged. Co-Authored-By: Claude Opus 4.7 --- .../skills/dandi-linkml-validation-report/SKILL.md | 6 +++--- .../references/DESIGN.md | 2 +- .../references/OUTPUT.md | 4 ++-- .../scripts/generate_report.py | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.claude/skills/dandi-linkml-validation-report/SKILL.md b/.claude/skills/dandi-linkml-validation-report/SKILL.md index d2ed558b..519fa48f 100644 --- a/.claude/skills/dandi-linkml-validation-report/SKILL.md +++ b/.claude/skills/dandi-linkml-validation-report/SKILL.md @@ -1,6 +1,6 @@ --- name: dandi-linkml-validation-report -description: Generate a Markdown report assessing how `dandischema/models.yaml` (the LinkML schema) validates against real DANDI Archive Dandiset metadata. Use when the user wants to assess schema fitness across the archive, investigate a class of validation failure across many dandisets, or compare before/after for a schema change. Covers fetching raw metadata for every dandiset (draft + every published version), running closed-world JSON-schema validation via the LinkML Python API, and aggregating per-version results into a top-level REPORT.md bucketed by target class (Dandiset / PublishedDandiset) × schemaVersion. +description: Generate a Markdown report assessing how `dandischema/models.yaml` (the LinkML schema) validates against real DANDI Archive Dandiset metadata. Use when the user wants to assess schema fitness across the archive, investigate a class of validation failure across many dandisets, or compare before/after for a schema change. Covers fetching raw metadata for every dandiset (draft + every published version), running closed-world JSON-schema validation via the LinkML Python API, and aggregating per-version results into a top-level README.md bucketed by target class (Dandiset / PublishedDandiset) × schemaVersion. compatibility: Requires the `linkml-auto-converted` hatch env defined in this repo's pyproject.toml (provides linkml, linkml-runtime, dandi, typer) and network access to a DANDI Archive instance. allowed-tools: Bash(git:*) Bash(hatch:*) Read --- @@ -82,14 +82,14 @@ hatch run linkml-auto-converted:python \ --commit-date $(git show -s --format=%cI linkml-auto-converted) ``` -Writes `$ROOT/REPORT.md`: overall counts, then per-bucket tables +Writes `$ROOT/README.md`: overall counts, then per-bucket tables (target class × schemaVersion) with top error patterns and links to each version's `SUMMARY.md`. ## Further reading - [references/OUTPUT.md](references/OUTPUT.md) — directory layout, - `info.json` / `validation.json` field shapes, REPORT.md structure. + `info.json` / `validation.json` field shapes, README.md structure. - [references/DESIGN.md](references/DESIGN.md) — key design choices: closed-world JSON-schema plugin, `@context` strip, byte-equivalent CLI transcript, all-or-nothing fetch writes, resume semantics. diff --git a/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md b/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md index 3c08ee32..9734afd9 100644 --- a/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md +++ b/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md @@ -129,7 +129,7 @@ Each stage skips work it's already done: - `validate_metadata.py`: skip a version when `validation.json` exists and parses, unless `--refresh`. If it parses but is unreadable, log a warning and re-validate. -- `generate_report.py`: stateless; always rewrites `REPORT.md` from +- `generate_report.py`: stateless; always rewrites `README.md` from whatever `validation.json` files are present. Failures in either of the first two stages are caught at the version diff --git a/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md b/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md index 70fde856..443fd70b 100644 --- a/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md +++ b/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md @@ -8,7 +8,7 @@ and the shape of every JSON artifact. The top-level `SKILL.md` describes ``` linkml-validation-reports// -├── REPORT.md # written by generate_report.py +├── README.md # written by generate_report.py └── data/ └── / # e.g. 000003 ├── draft/ # draft version @@ -135,7 +135,7 @@ lists target class, archive status, modified timestamp, raw inlines up to the first 20 problem lines and links back to the raw files. -## `REPORT.md` +## `README.md` Top-level report written by `generate_report.py`. diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py index b657ca96..2ae64838 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -4,7 +4,7 @@ The report directory is expected to look like:: linkml-validation-reports// - ├── REPORT.md <-- written by this script + ├── README.md <-- written by this script └── data/ ├── 000003/ │ ├── draft/ @@ -17,7 +17,7 @@ └── 000004/... For each ``validation.json`` produced by ``validate_metadata.py``, this -script aggregates results and writes a single ``REPORT.md`` at the top +script aggregates results and writes a single ``README.md`` at the top of the report directory. The report contains: @@ -100,7 +100,7 @@ def _load_records(data_dir: Path) -> list[dict]: except json.JSONDecodeError: logger.warning("skipping unreadable %s", vj) continue - # Path to per-version SUMMARY.md, relative to REPORT.md (which + # Path to per-version SUMMARY.md, relative to README.md (which # sits one level above ``data_dir``). rec["_summary_link"] = ( f"data/{vj.parent.parent.name}/{vj.parent.name}/SUMMARY.md" @@ -210,7 +210,7 @@ def _render_report( schema: str, top_n_patterns: int, ) -> None: - """Write the top-level ``REPORT.md`` based on ``records``.""" + """Write the top-level ``README.md`` based on ``records``.""" # Group records into the (class, schemaVersion) buckets the report # is organized around. buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) @@ -306,7 +306,7 @@ def main( ), log_level: str = typer.Option("INFO", "--log-level", "-l"), ) -> None: - """Aggregate per-version validation outputs into a top-level REPORT.md.""" + """Aggregate per-version validation outputs into a top-level README.md.""" logging.basicConfig( format="[%(asctime)s] %(levelname)s %(name)s: %(message)s", level=getattr(logging, log_level.upper()), @@ -320,7 +320,7 @@ def main( _attach_info(records, data_dir) logger.info("loaded %d validation records", len(records)) - out_path = report_root / "REPORT.md" + out_path = report_root / "README.md" _render_report( out_path, records, From 36863e26d4a6adaebb49f6c676ce341b4d4e5119 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 17:17:49 -0700 Subject: [PATCH 53/90] Drop references/ from the skill, lean on script docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `references/DESIGN.md` and `references/OUTPUT.md` largely duplicated content already present in the three scripts' module docstrings and inline comments — the maintenance cost of two parallel sources outweighed the progressive-disclosure benefit at this scale. Anyone needing the on-disk layout or design rationale can read the relevant script directly. SKILL.md's "Further reading" section is replaced with a one-line pointer to the script docstrings. Co-Authored-By: Claude Opus 4.7 --- .../dandi-linkml-validation-report/SKILL.md | 9 +- .../references/DESIGN.md | 136 --------------- .../references/OUTPUT.md | 165 ------------------ 3 files changed, 2 insertions(+), 308 deletions(-) delete mode 100644 .claude/skills/dandi-linkml-validation-report/references/DESIGN.md delete mode 100644 .claude/skills/dandi-linkml-validation-report/references/OUTPUT.md diff --git a/.claude/skills/dandi-linkml-validation-report/SKILL.md b/.claude/skills/dandi-linkml-validation-report/SKILL.md index 519fa48f..aaa02fae 100644 --- a/.claude/skills/dandi-linkml-validation-report/SKILL.md +++ b/.claude/skills/dandi-linkml-validation-report/SKILL.md @@ -86,10 +86,5 @@ Writes `$ROOT/README.md`: overall counts, then per-bucket tables (target class × schemaVersion) with top error patterns and links to each version's `SUMMARY.md`. -## Further reading - -- [references/OUTPUT.md](references/OUTPUT.md) — directory layout, - `info.json` / `validation.json` field shapes, README.md structure. -- [references/DESIGN.md](references/DESIGN.md) — key design choices: - closed-world JSON-schema plugin, `@context` strip, byte-equivalent - CLI transcript, all-or-nothing fetch writes, resume semantics. +For details on the on-disk layout, JSON field shapes, and design +rationale, read the module docstrings of the three scripts directly. diff --git a/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md b/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md deleted file mode 100644 index 9734afd9..00000000 --- a/.claude/skills/dandi-linkml-validation-report/references/DESIGN.md +++ /dev/null @@ -1,136 +0,0 @@ -# Design notes - -Rationale for non-obvious choices in the pipeline. Read this when you're -modifying a script and want to know why something was done a particular -way; the SKILL.md and OUTPUT.md cover the *what*. - -## LinkML Python API, not the CLI - -`validate_metadata.py` drives the `linkml.validator.Validator` class -directly rather than shelling out to `linkml-validate`. Reasons: - -- One validation run produces both the structured `validation.json` - and the human-readable `validation.txt`. Shelling out would force - either a second run or downstream regex parsing of CLI text. -- `linkml-validate` has no JSON output flag (verified against the - installed `linkml/validator/cli.py`), and its `--config` only - externalizes the same arguments — there is no formatter knob. -- Subprocess overhead is meaningful when iterating ~900 dandisets - with a few thousand version directories. - -`Validator` is constructed once and reused across every version -directory; schema parsing is the expensive part. - -## Closed-world JSON-schema plugin - -We pass `JsonschemaValidationPlugin(closed=True)` when constructing the -validator. That matches the CLI's default (`{"JsonschemaValidationPlugin": -{"closed": True}}` in `linkml/validator/cli.py`), so the results we -collect are the same ones `linkml-validate` would emit. - -`closed=True` causes LinkML to emit `additionalProperties: false` in -the generated JSON Schema — extra keys on a class become validation -errors. Switching to `closed=False` would silently accept unknown -fields and mask real bugs (e.g. `includeInCitation` on `Affiliation`), -so it's not an acceptable knob for "ignore one specific extra key". - -## `@context` is stripped before validation - -DANDI metadata carries a top-level JSON-LD framing key (`@context`) -that isn't part of the `Dandiset` / `PublishedDandiset` LinkML class -definitions. With `closed=True` it shows up as an -`additionalProperties` violation — see linkml/linkml#3442. - -We `raw.pop("@context", None)` on the loaded dict before passing it to -`validator.validate(...)`. Cheap, local to the validator step, and -doesn't touch the saved `metadata.json` (which we keep verbatim). -Considered alternatives: - -- Open up `closed=False` globally — too broad; loses signal. -- Post-validation filter — runs the validator, gets a known-noise - result, throws it away. Backwards from intent. -- Custom plugin subclass — tracking LinkML internals just to skip one - message; not worth it. -- Schema patch in `models.yaml` — `models.yaml` is auto-generated from - `dandischema.models`, so the change would have to live in the - Pydantic source. Too far away from the narrow concern. - -## CLI byte-equivalent transcript - -`validation.txt` is rendered by formatting each `ValidationResult` -with the same f-string `linkml-validate` uses -(`linkml/validator/cli.py`): - -``` -[] [/] -``` - -Plus a single-line `No issues found` when zero results, plus an -`exit_code` field in the JSON header that mirrors the CLI's exit code -(1 iff any `ERROR` severity is present, else 0). We interpolate -`r.instance_index` directly with no fallback so a `None` (which would -appear if LinkML stops emitting `0` for non-list instances) renders as -the literal `"None"` — same as the CLI. Today the JSON-schema plugin -always emits `0` for our single-instance `validate(raw, ...)` calls, -so the field renders as `0` in practice. - -## Structured problem records - -`validation.json` does *not* persist the CLI text and ask the report -to parse it. Each problem is the dict-form of a `ValidationResult` -plus the bits of `source` that matter for grouping -(`source.validator`, `source.validator_value`). - -`source` is excluded from default pydantic serialization (it's a -plugin-defined object), so `_result_to_dict` extracts those two -attributes by hand. The `instance` field — which echoes the full data -instance back into every result — is excluded to avoid duplicating -`metadata.json` per problem. - -`generate_report.py` groups patterns by `[] <> -`, drawing the validator keyword from the structured record. -That groups across dandisets without regex-scrubbing the per-file path -out of the CLI text, and surfaces the failing JSON-schema keyword -(e.g. `additionalProperties`, `enum`, `format`) directly in the -report. - -## All-or-nothing fetch writes - -`fetch_metadata.py` writes `metadata.json` and `info.json` together or -not at all. Mechanism: - -1. Network calls first — `get_raw_metadata()` and `get_version()` both - complete before any filesystem write. -2. Pre-render both JSON strings. -3. Write each to a `.tmp` sibling. -4. `os.replace(...)` each `.tmp` onto its final path. - -`os.replace` performs the POSIX `rename(2)` syscall; either the -destination ends up at the new content or it stays as it was before. -That guarantee is what makes the resume guard correct: the next run -checks for *both* `metadata.json` and `info.json` and skips only if -both exist, so a process killed mid-fetch never leaves a half-written -file that the resume guard would mistake for completion. (Leftover -`.tmp` files are harmless cruft that the next successful fetch -overwrites.) - -We considered a `try/except` around plain writes to the final paths. -That's strictly worse: a SIGKILL between writing `metadata.json` fully -and `info.json` partially leaves both files existing with `info.json` -truncated, and the resume guard would skip the retry. The temp+rename -pattern can't reach that state. - -## Resume semantics - -Each stage skips work it's already done: - -- `fetch_metadata.py`: skip a version when both `metadata.json` and - `info.json` exist, unless `--refresh`. -- `validate_metadata.py`: skip a version when `validation.json` - exists and parses, unless `--refresh`. If it parses but is - unreadable, log a warning and re-validate. -- `generate_report.py`: stateless; always rewrites `README.md` from - whatever `validation.json` files are present. - -Failures in either of the first two stages are caught at the version -level — one broken dandiset doesn't abort the run. diff --git a/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md b/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md deleted file mode 100644 index 443fd70b..00000000 --- a/.claude/skills/dandi-linkml-validation-report/references/OUTPUT.md +++ /dev/null @@ -1,165 +0,0 @@ -# Output anatomy - -Reference for the on-disk layout produced by the three pipeline stages -and the shape of every JSON artifact. The top-level `SKILL.md` describes -*how* to run the pipeline; this file describes *what it produces*. - -## Directory layout - -``` -linkml-validation-reports// -├── README.md # written by generate_report.py -└── data/ - └── / # e.g. 000003 - ├── draft/ # draft version - │ ├── metadata.json # written by fetch_metadata.py - │ ├── info.json # written by fetch_metadata.py - │ ├── validation.json # written by validate_metadata.py - │ ├── validation.txt # written by validate_metadata.py - │ └── SUMMARY.md # written by validate_metadata.py - └── / # e.g. 0.230629.1955 - └── … # same five files -``` - -`` is whatever the caller of `generate_report.py` passed for -`--commit-hash` (typically `git rev-parse --short=7 linkml-auto-converted`). -The pipeline doesn't inspect or enforce it — it's just a namespace for -the run. - -## `metadata.json` - -Verbatim raw metadata as returned by `RemoteDandiset.get_raw_metadata()`. -Pretty-printed JSON. Not modified after the fetch — every downstream -consumer reads this as the authoritative archive payload. - -## `info.json` - -Side-car written alongside `metadata.json` so the validator and the -report can pick the right target class and group records without -re-querying the archive. - -```json -{ - "dandiset_id": "000003", - "version": "0.230629.1955", - "is_published": true, - "status": "VALID", - "modified": "2023-06-29T19:55:35.080882+00:00", - "schema_version": "0.6.4" -} -``` - -| Field | Source | -|------------------|------------------------------------------------------| -| `dandiset_id` | `RemoteDandiset.identifier` | -| `version` | Version id, or the literal string `"draft"` | -| `is_published` | `False` for the draft version, `True` otherwise | -| `status` | `version_info.status.value` (DANDI `VersionStatus`) | -| `modified` | `version_info.modified.isoformat()` (always present) | -| `schema_version` | `metadata["schemaVersion"]` (may be `null`) | - -`metadata.json` and `info.json` are written all-or-nothing: each is -written to a `.tmp` sibling first, then `os.replace`'d into place. If -the fetch is killed mid-write, neither final path appears, so the -resume guard treats that version as unfetched. - -## `validation.json` - -Structured validation record written by `validate_metadata.py`. One -record per version directory: - -```json -{ - "dandiset_id": "000003", - "version": "draft", - "is_published": false, - "target_class": "Dandiset", - "schema_version": "0.6.0", - "exit_code": 1, - "problem_count": 3, - "problems": [ { … }, … ] -} -``` - -Each entry in `problems` is a JSON-friendly serialization of one -LinkML `ValidationResult`: - -```json -{ - "type": "jsonschema validation", - "severity": "ERROR", - "message": "'Dandiset' is not one of ['PublishedDandiset'] in /schemaKey", - "instance_index": 0, - "instantiates": "PublishedDandiset", - "context": [], - "source": { - "validator": "enum", - "validator_value": ["PublishedDandiset"] - } -} -``` - -Notes on the fields: - -- `instance` (the full data instance the validator was given) is - excluded — it would duplicate `metadata.json` per problem and bloat - the file. Look at the sibling `metadata.json` if you need the - payload. -- `source.validator` / `source.validator_value` come from the - `JsonschemaValidationPlugin` and identify the failing JSON-schema - keyword (e.g. `additionalProperties`, `enum`, `format`, - `required`). The report generator groups patterns by this field so - similar failures across dandisets count together. -- `exit_code` is `1` if any `ERROR`-severity result is present, else - `0` — matches `linkml-validate`'s exit-code semantics. - -## `validation.txt` - -Byte-equivalent to what `linkml-validate -s -C -` would have printed. Each line: - -``` -[] [/] -``` - -When there are zero results, the file contains the single line -`No issues found` (matching the CLI's success banner). The literal -text comes from re-formatting `ValidationResult` objects — we don't -shell out to the CLI. - -## `SUMMARY.md` - -Per-version markdown summary linked from the top-level report. Header -lists target class, archive status, modified timestamp, raw -`schemaVersion`, equivalent CLI exit code, and problem count; body -inlines up to the first 20 problem lines and links back to the raw -files. - -## `README.md` - -Top-level report written by `generate_report.py`. - -Header: - -- Branch name (default `linkml-auto-converted`) -- Commit hash + ISO commit date (passed in via flags) -- Schema path (default `dandischema/models.yaml`) -- Total versions and overall valid/with-problems count - -Then one section per target class: - -1. `Draft versions (target class: Dandiset)` -2. `Published versions (target class: PublishedDandiset)` - -Each section is sub-bucketed by `schemaVersion` (newest first; -`` last when present). Each sub-bucket contains: - -- A one-line headline (versions / valid / failing) -- Top-N grouped problem patterns of the form - `[] <> ` with counts -- A per-version table: `Dandiset | Version | Problems | API Status | - Modified`, where `Problems` links to each version's `SUMMARY.md`. - -The "API Status" column is the per-version `status` field that DANDI's -own API returns (`VALID`, `Published`, `Pending`, etc.) — not anything -the LinkML validator computed. From 2f667eed17dfb3aaae06f93ee52781ab3c5e8ea3 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 18:48:52 -0700 Subject: [PATCH 54/90] Migrate metadata before validating; flag migration failures in report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `validate_metadata.py` now runs ``dandischema.metadata.migrate`` on each version's raw metadata before validating. The migrated instance is persisted as ``metadata_migrated.json`` (the verbatim ``metadata.json`` is left untouched) and is what the LinkML validator sees. Versions whose migration fails are recorded with the error and skipped for validation — the validator never sees something the migrator couldn't handle. `validation.json` gains ``migration_status`` (``"success"`` / ``"failed"``) and ``migration_error`` fields. On migration failure ``problems`` is empty and ``exit_code`` is null. The CLI-equivalent transcript is replaced by a one-line ``Migration failed: …`` notice and ``SUMMARY.md`` calls the failure out instead of rendering a problems block. `generate_report.py` distinguishes migration failures in both the overall headline (``N valid / M migration-failed / P with problems``) and per-bucket sections (extra ``Migration failed:`` count when non-zero) and renders an ``[migration failed]`` cell in the per-version table linking to the version's ``SUMMARY.md`` for the failure detail. Migration-failed versions are excluded from problem pattern grouping since validation never ran for them. Co-Authored-By: Claude Opus 4.7 --- .../dandi-linkml-validation-report/SKILL.md | 18 +- .../scripts/generate_report.py | 65 +++- .../scripts/validate_metadata.py | 299 +++++++++++++----- 3 files changed, 270 insertions(+), 112 deletions(-) diff --git a/.claude/skills/dandi-linkml-validation-report/SKILL.md b/.claude/skills/dandi-linkml-validation-report/SKILL.md index aaa02fae..cd1365cd 100644 --- a/.claude/skills/dandi-linkml-validation-report/SKILL.md +++ b/.claude/skills/dandi-linkml-validation-report/SKILL.md @@ -1,6 +1,6 @@ --- name: dandi-linkml-validation-report -description: Generate a Markdown report assessing how `dandischema/models.yaml` (the LinkML schema) validates against real DANDI Archive Dandiset metadata. Use when the user wants to assess schema fitness across the archive, investigate a class of validation failure across many dandisets, or compare before/after for a schema change. Covers fetching raw metadata for every dandiset (draft + every published version), running closed-world JSON-schema validation via the LinkML Python API, and aggregating per-version results into a top-level README.md bucketed by target class (Dandiset / PublishedDandiset) × schemaVersion. +description: Generate a Markdown report assessing how `dandischema/models.yaml` (the LinkML schema) validates against real DANDI Archive Dandiset metadata after migrating each instance to the latest schema version. Use when the user wants to assess schema fitness across the archive, investigate a class of validation failure across many dandisets, or compare before/after for a schema change. Covers fetching raw metadata for every dandiset (draft + every published version), migrating each instance via `dandischema.metadata.migrate`, running closed-world JSON-schema validation on successfully-migrated instances via the LinkML Python API, and aggregating per-version results into a top-level README.md bucketed by target class (Dandiset / PublishedDandiset) × schemaVersion. Versions whose metadata can't be migrated are flagged in the report; validation is skipped for them. compatibility: Requires the `linkml-auto-converted` hatch env defined in this repo's pyproject.toml (provides linkml, linkml-runtime, dandi, typer) and network access to a DANDI Archive instance. allowed-tools: Bash(git:*) Bash(hatch:*) Read --- @@ -57,7 +57,7 @@ Re-running is safe — already-downloaded versions are skipped unless `--refresh` is passed. `--limit N` truncates to N dandisets for smoke tests. `-i ` selects a non-production DANDI instance. -### 2. Validate +### 2. Migrate + validate ```sh hatch run linkml-auto-converted:python \ @@ -65,12 +65,14 @@ hatch run linkml-auto-converted:python \ $ROOT/data --schema dandischema/models.yaml ``` -For each version directory, runs LinkML's `Validator` Python API once -and writes `validation.json` (structured results), `validation.txt` -(byte-equivalent to `linkml-validate` CLI output), and `SUMMARY.md`. -Drafts are validated against `Dandiset`; published versions against -`PublishedDandiset`. `--refresh` re-validates already-validated -versions. +For each version directory, runs `dandischema.metadata.migrate` on +the raw metadata first, then validates the migrated instance against +the LinkML schema (drafts → `Dandiset`, published → `PublishedDandiset`). +Writes `metadata_migrated.json` (when migration succeeds), plus +`validation.json` (structured record carrying `migration_status`), +`validation.txt`, and `SUMMARY.md`. Versions whose migration fails +are recorded with the error and skipped for validation. `--refresh` +re-runs migration and validation on already-processed versions. ### 3. Generate report diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py index 2ae64838..e51941f9 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -125,6 +125,11 @@ def _bucket_key(rec: dict) -> tuple[str, str]: # --------------------------------------------------------------------------- +def _migration_failed(rec: dict) -> bool: + """True if this record's metadata could not be migrated.""" + return rec.get("migration_status") == "failed" + + def _render_bucket( fh, title: str, @@ -135,20 +140,33 @@ def _render_bucket( """Render one bucket section to ``fh``. Emits: - * a one-line headline counting versions / valid / failing, - * a "top error patterns" list, + * a one-line headline counting versions / migration-failed / + valid / failing, + * a "top error patterns" list (validation problems only — + migration-failed versions never reached the validator), * a table indexing every version with a link to its per-version ``SUMMARY.md``. """ n_total = len(records) - n_valid = sum(1 for r in records if r["problem_count"] == 0) - n_failing = n_total - n_valid + n_mig_failed = sum(1 for r in records if _migration_failed(r)) + n_validated = n_total - n_mig_failed + # "Valid" here means migration succeeded *and* validation found no + # problems. Migration-failed versions are excluded from both + # ``valid`` and ``with-problems`` since validation never ran. + n_valid = sum( + 1 for r in records if not _migration_failed(r) and r["problem_count"] == 0 + ) + n_with_problems = n_validated - n_valid fh.write(f"### {title}\n\n") - fh.write( - f"- **Versions:** {n_total} • " - f"**Valid:** {n_valid} • **With problems:** {n_failing}\n\n" - ) + headline_parts = [f"**Versions:** {n_total}"] + if n_mig_failed: + headline_parts.append(f"**Migration failed:** {n_mig_failed}") + headline_parts += [ + f"**Valid:** {n_valid}", + f"**With problems:** {n_with_problems}", + ] + fh.write("- " + " • ".join(headline_parts) + "\n\n") # --- Top error patterns within this bucket. --- pattern_counter: Counter[str] = Counter() @@ -167,11 +185,16 @@ def _render_bucket( fh.write("| Dandiset | Version | Problems | API Status | Modified |\n") fh.write("|---|---|---:|---|---|\n") for r in sorted(records, key=lambda x: (x["dandiset_id"], x["version"])): - problems_cell = ( - f"[{r['problem_count']}]({r['_summary_link']})" - if r["problem_count"] - else f"[OK]({r['_summary_link']})" - ) + if _migration_failed(r): + # Migration-failed versions don't have a problem count to + # display. Render a distinct cell so the reader can spot + # them at a glance and click through to the per-version + # SUMMARY.md for the migration error. + problems_cell = f"[migration failed]({r['_summary_link']})" + elif r["problem_count"]: + problems_cell = f"[{r['problem_count']}]({r['_summary_link']})" + else: + problems_cell = f"[OK]({r['_summary_link']})" # ``status`` and ``modified`` come from each version's ``info.json``; # ``_attach_info`` has already stashed them onto the record so we # can render the table without touching the filesystem here. @@ -225,11 +248,19 @@ def _render_report( fh.write(f"- **Schema:** `{schema}`\n") fh.write(f"- **Total dandiset versions checked:** {len(records)}\n\n") - n_valid = sum(1 for r in records if r["problem_count"] == 0) + n_total = len(records) + n_mig_failed = sum(1 for r in records if _migration_failed(r)) + n_valid = sum( + 1 for r in records if not _migration_failed(r) and r["problem_count"] == 0 + ) + n_with_problems = n_total - n_mig_failed - n_valid + overall_parts = [f"{n_valid} valid"] + if n_mig_failed: + overall_parts.append(f"{n_mig_failed} migration-failed") + overall_parts.append(f"{n_with_problems} with problems") fh.write( - f"**Overall:** {n_valid} valid / " - f"{len(records) - n_valid} with problems " - f"out of {len(records)} versions.\n\n" + f"**Overall:** {' / '.join(overall_parts)} " + f"out of {n_total} versions.\n\n" ) # Draft section first (target class: Dandiset), then published. diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py index bbc8f282..24fffe54 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py @@ -1,26 +1,48 @@ #!/usr/bin/env python3 -"""Validate downloaded dandiset metadata using the LinkML validator Python API. +"""Migrate then validate downloaded dandiset metadata. For every dandiset version directory produced by ``fetch_metadata.py`` this -script runs LinkML validation against ``dandischema/models.yaml`` and emits -three sibling files in the version directory: - - ``validation.json`` — machine-readable record. Wraps the structured - ``ValidationResult`` objects from the validator - (severity, message, instance index, JSON-pointer - path, validator name, etc.) plus a small header - identifying the dandiset/version/target class. - - ``validation.txt`` — human-readable transcript that is byte-equivalent - to what ``linkml-validate`` would have printed for - the same inputs (same ``[severity] [source/idx] - message`` template, same ``No issues found`` - banner, same exit-code semantics). One run, two - outputs — we don't shell out to the CLI. - - ``SUMMARY.md`` — short markdown summary of the version's - validation outcome (linked from the top-level - report). +script: + + 1. Migrates the raw metadata to the latest ``Dandiset`` / + ``PublishedDandiset`` schema using + ``dandischema.metadata.migrate(skip_validation=True)``. Migration + can fail (the source metadata may be malformed in ways the migrator + can't handle) — that's recorded on the version and validation is + skipped for it. + + 2. For successfully migrated metadata, runs LinkML validation against + ``dandischema/models.yaml`` using the LinkML ``Validator`` Python + API. + +Each version directory ends up with these sibling files: + + ``metadata_migrated.json`` — the migrated metadata, written only + when migration succeeds. Verbatim + ``metadata.json`` is preserved + untouched. + + ``validation.json`` — machine-readable record. Always + written. Carries a ``migration_status`` + field (``"success"`` or ``"failed"``). + On success, the structured LinkML + ``ValidationResult`` objects (severity, + message, JSON-pointer path, validator + keyword, etc.) are stored in the + ``problems`` array. On migration + failure, ``problems`` is empty. + + ``validation.txt`` — human-readable transcript. On success, + byte-equivalent to what ``linkml-validate`` + would have printed (same ``[severity] + [source/idx] message`` template, same + ``No issues found`` banner). On + migration failure, a one-line + ``Migration failed: …`` notice instead. + + ``SUMMARY.md`` — short markdown summary of the version's + outcome (linked from the top-level + report). The target class is decided from the version's ``info.json``: @@ -28,8 +50,8 @@ * ``PublishedDandiset`` — for any non-``draft`` (i.e. published) version Re-running is safe: by default versions that already have a -``validation.json`` are skipped. Pass ``--refresh`` to re-validate -everything. +``validation.json`` are skipped. Pass ``--refresh`` to re-run migration +and validation for every version. Example ------- @@ -43,6 +65,7 @@ import json import logging +import os from pathlib import Path from linkml.validator import Validator @@ -50,6 +73,8 @@ from linkml.validator.report import Severity, ValidationResult import typer +from dandischema.metadata import migrate + logger = logging.getLogger("validate_metadata") app = typer.Typer(add_completion=False, help=__doc__.splitlines()[0]) @@ -98,26 +123,42 @@ def _format_cli_line(r: ValidationResult, source_label: str) -> str: return f"[{r.severity.value}] [{source_label}/{r.instance_index}] {r.message}" +def _atomic_write_json(path: Path, data: object) -> None: + """Write ``data`` to ``path`` as pretty JSON via temp + ``os.replace``. + + Same all-or-nothing pattern as ``fetch_metadata.py``: if anything + goes wrong before the rename, no file ends up at ``path``. + """ + text = json.dumps(data, indent=2) + "\n" + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text) + os.replace(tmp, path) + + # --------------------------------------------------------------------------- -# Per-directory validation +# Per-directory migration + validation # --------------------------------------------------------------------------- def _validate_one( validator: Validator, version_dir: Path, *, refresh: bool -) -> tuple[str, int]: - """Validate one ``/`` directory. +) -> tuple[str, str, int]: + """Migrate then validate one ``/`` directory. Reads ``info.json`` (written by ``fetch_metadata.py``) to decide the - target class, runs the validator on ``metadata.json``, and writes - ``validation.json`` / ``validation.txt`` / ``SUMMARY.md`` alongside - it. - - Returns ``(target_class, n_results)`` so the caller can log a tally - without re-reading the file. + target class, attempts a migration of ``metadata.json``, and — when + migration succeeds — validates the migrated metadata against the + LinkML schema. Writes ``metadata_migrated.json`` (on success), + ``validation.json`` / ``validation.txt`` / ``SUMMARY.md``. + + Returns ``(target_class, migration_status, n_results)`` so the + caller can log a tally without re-reading the file. + ``migration_status`` is ``"success"`` or ``"failed"``; + ``n_results`` is ``0`` when migration failed. """ metadata_file = version_dir / "metadata.json" info_file = version_dir / "info.json" + migrated_metadata_file = version_dir / "metadata_migrated.json" out_text = version_dir / "validation.txt" out_json = version_dir / "validation.json" out_md = version_dir / "SUMMARY.md" @@ -131,41 +172,91 @@ def _validate_one( if out_json.exists() and not refresh: try: existing = json.loads(out_json.read_text()) - return target_class, int(existing.get("problem_count", 0)) + problem_count = int(existing.get("problem_count", 0)) except (json.JSONDecodeError, ValueError): logger.warning( "re-validating %s — existing validation.json is unreadable", version_dir, ) + else: + return ( + target_class, + existing.get("migration_status", "success"), + problem_count, + ) - # --- Run validation once. --- raw = json.loads(metadata_file.read_text()) # ``@context`` is a JSON-LD framing field that's not part of the # ``Dandiset`` / ``PublishedDandiset`` LinkML class definitions, so a # closed-world JSON-schema check flags it as an unexpected property - # (see linkml/linkml#3442). Strip it before validation so we don't - # drown the report in noise that has nothing to do with the model. + # (see linkml/linkml#3442). Strip it before migration/validation so + # we don't drown the report in noise that has nothing to do with the + # model. raw.pop("@context", None) - report = validator.validate(raw, target_class=target_class) - results: list[ValidationResult] = report.results - - # ``linkml-validate``'s exit code is 1 iff any ERROR-severity result - # is present, else 0. We replicate that for downstream consumers - # that key off ``exit_code``. - has_error = any(r.severity is Severity.ERROR for r in results) - exit_code = 1 if has_error else 0 - - # --- Render the human-readable transcript (CLI-equivalent). --- - # The CLI prints ``loader.source`` as the bracketed path; for a - # file-backed JsonLoader that's the file path string, so use the - # same here. - source_label = str(metadata_file) - if results: - transcript_lines = [_format_cli_line(r, source_label) for r in results] + + # --- Migration step. --- + # ``skip_validation=True`` keeps ``migrate`` from running its own + # internal Pydantic validation; we want to validate against the + # *LinkML* schema afterward, and we don't want a Pydantic failure to + # mask a successful structural migration. + migration_status: str + migration_error: str | None + migrated: dict | None + try: + migrated = migrate(raw, skip_validation=True) + except Exception as e: + # Migration helpers raise ``NotImplementedError`` / + # ``ValueError`` for known unsupported inputs, but ``migrate`` + # also rewires Pydantic-level traversals where any number of + # other errors can surface. Catch broadly so one bad version + # doesn't abort the run. + migrated = None + migration_status = "failed" + migration_error = repr(e) + else: + migration_status = "success" + migration_error = None + + # --- Branch on migration outcome. --- + results: list[ValidationResult] + transcript_lines: list[str] + exit_code: int | None + + if migration_status == "success": + assert migrated is not None + + # Persist the migrated metadata so the report can link to it + # and the user can inspect what was actually validated. + _atomic_write_json(migrated_metadata_file, migrated) + + report = validator.validate(migrated, target_class=target_class) + results = report.results + + # ``linkml-validate``'s exit code is 1 iff any ERROR-severity + # result is present, else 0. Replicate that for downstream + # consumers that key off ``exit_code``. + has_error = any(r.severity is Severity.ERROR for r in results) + exit_code = 1 if has_error else 0 + + # The CLI prints ``loader.source`` as the bracketed path; for a + # file-backed JsonLoader that's the file path string. Point + # readers at the migrated file since that's what was actually + # validated. + source_label = str(migrated_metadata_file) + if results: + transcript_lines = [_format_cli_line(r, source_label) for r in results] + else: + # Mirrors the CLI's success banner so byte-equivalence holds + # in the zero-results case too. + transcript_lines = ["No issues found"] else: - # Mirrors the CLI's success banner so byte-equivalence holds in - # the zero-results case too. - transcript_lines = ["No issues found"] + # Migration failed — leave any prior ``metadata_migrated.json`` + # alone (it would belong to a previous successful run) and + # don't try to validate. + results = [] + exit_code = None + transcript_lines = [f"Migration failed: {migration_error}"] + out_text.write_text("\n".join(transcript_lines) + "\n") # --- Persist the structured record. --- @@ -175,46 +266,73 @@ def _validate_one( "is_published": is_published, "target_class": target_class, "schema_version": info.get("schema_version"), + "migration_status": migration_status, + "migration_error": migration_error, "exit_code": exit_code, "problem_count": len(results), "problems": [_result_to_dict(r) for r in results], } - out_json.write_text(json.dumps(record, indent=2) + "\n") + _atomic_write_json(out_json, record) # --- Per-version markdown summary, linked from the top-level report. --- md_lines = [ f"# Validation summary — {info['dandiset_id']} @ {info['version']}", "", f"- **Target class:** `{target_class}`", - f"- **Status:** {info.get('status')}", + f"- **API status:** {info.get('status')}", f"- **Modified:** {info.get('modified')}", - f"- **schemaVersion:** {info.get('schema_version')}", - f"- **Equivalent `linkml-validate` exit code:** {exit_code}", - f"- **# problems:** {len(results)}", - "", - "## Files", - "", - "- [`metadata.json`](metadata.json) — raw metadata as fetched from the archive", - "- [`validation.txt`](validation.txt) — `linkml-validate`-equivalent transcript", - "- [`validation.json`](validation.json) — structured validation record", - "", + f"- **Source schemaVersion:** {info.get('schema_version')}", + f"- **Migration status:** `{migration_status}`", ] - if results: + if migration_status == "success": + md_lines += [ + f"- **Equivalent `linkml-validate` exit code:** {exit_code}", + f"- **# problems:** {len(results)}", + "", + "## Files", + "", + "- [`metadata.json`](metadata.json) — raw metadata as fetched from the archive", + "- [`metadata_migrated.json`](metadata_migrated.json)" + " — metadata after migration to the latest schema", + "- [`validation.txt`](validation.txt) — `linkml-validate`-equivalent transcript", + "- [`validation.json`](validation.json) — structured validation record", + "", + ] + if results: + md_lines += [ + "## First 20 problems", + "", + "```", + *transcript_lines[:20], + "```", + ] + if len(results) > 20: + md_lines.append( + f"_… {len(results) - 20} more — see " + "[`validation.txt`](validation.txt)._" + ) + else: md_lines += [ - "## First 20 problems", + "", + "## Migration failure", + "", + "Validation was **not** run because the metadata could not be", + "migrated to the latest schema version.", "", "```", - *transcript_lines[:20], + f"{migration_error}", "```", + "", + "## Files", + "", + "- [`metadata.json`](metadata.json) — raw metadata as fetched from the archive", + "- [`validation.txt`](validation.txt) — migration-failure notice", + "- [`validation.json`](validation.json) — structured record (no validation results)", + "", ] - if len(results) > 20: - md_lines.append( - f"_… {len(results) - 20} more — see " - "[`validation.txt`](validation.txt)._" - ) out_md.write_text("\n".join(md_lines) + "\n") - return target_class, len(results) + return target_class, migration_status, len(results) # --------------------------------------------------------------------------- @@ -237,11 +355,11 @@ def main( refresh: bool = typer.Option( False, "--refresh", - help="Re-validate even when validation.json already exists.", + help="Re-migrate and re-validate even when validation.json already exists.", ), log_level: str = typer.Option("INFO", "--log-level", "-l"), ) -> None: - """Validate every ``//metadata.json`` under ``root``.""" + """Migrate + validate every ``//metadata.json`` under ``root``.""" logging.basicConfig( format="[%(asctime)s] %(levelname)s %(name)s: %(message)s", level=getattr(logging, log_level.upper()), @@ -260,22 +378,29 @@ def main( version_dirs = sorted( p for p in root.glob("*/*") if p.is_dir() and (p / "metadata.json").is_file() ) - logger.info("found %d version directories to validate", len(version_dirs)) + logger.info("found %d version directories to process", len(version_dirs)) for vd in version_dirs: try: - target, n = _validate_one(validator, vd, refresh=refresh) + target, mig_status, n = _validate_one(validator, vd, refresh=refresh) except Exception as e: # Never let one broken dandiset abort the whole run. - logger.exception("validation failed for %s: %s", vd, e) + logger.exception("processing failed for %s: %s", vd, e) else: - logger.info( - "validated %s/%s as %s (%d problems)", - vd.parent.name, - vd.name, - target, - n, - ) + if mig_status == "success": + logger.info( + "%s/%s — migrated, validated as %s (%d problems)", + vd.parent.name, + vd.name, + target, + n, + ) + else: + logger.info( + "%s/%s — migration failed; validation skipped", + vd.parent.name, + vd.name, + ) if __name__ == "__main__": From 07e7d4794b57ce84f4e86ac9fd199283ea37ee0a Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 29 Apr 2026 23:58:52 -0700 Subject: [PATCH 55/90] Flat output layout; schema-hash-aware resume in validate_metadata.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The output tree no longer namespaces under a `/` directory. All artifacts live under one flat root: linkml-validation-reports/ ├── README.md └── data///{metadata.json, info.json, metadata_migrated.json, validation.json, validation.txt, SUMMARY.md} Raw metadata is schema-independent and only fetched once; subsequent runs against a different schema reuse it. `validate_metadata.py`'s resume guard is now schema-aware. Each `validation.json` is stamped with the SHA-256 of the schema file's bytes (`schema_sha256` field). On a re-run the guard skips a version only when its stamp matches the current schema, so a schema-content change — committed *or* uncommitted — re-runs migration and validation automatically without `--refresh`. `--refresh` is now documented as a forceful override only. Per-version logging moved into `_validate_one`. The function used to return a `(target_class, migration_status, n_problems)` tuple consumed *only* by the orchestrator's per-version log line. With the log emitted at the decision site (resumed / migration failed / migrated and validated), the return value carried no information and has been dropped. Co-Authored-By: Claude Opus 4.7 --- .../dandi-linkml-validation-report/SKILL.md | 35 ++++-- .../scripts/fetch_metadata.py | 2 +- .../scripts/generate_report.py | 7 +- .../scripts/validate_metadata.py | 111 ++++++++++++------ 4 files changed, 106 insertions(+), 49 deletions(-) diff --git a/.claude/skills/dandi-linkml-validation-report/SKILL.md b/.claude/skills/dandi-linkml-validation-report/SKILL.md index cd1365cd..a86ccbbe 100644 --- a/.claude/skills/dandi-linkml-validation-report/SKILL.md +++ b/.claude/skills/dandi-linkml-validation-report/SKILL.md @@ -36,13 +36,17 @@ idempotent and resumable. ## Workflow -Pick a short SHA of the schema commit to namespace the run: +The pipeline writes everything under one flat directory: ```sh -SHA=$(git rev-parse --short=7 linkml-auto-converted) -ROOT=linkml-validation-reports/$SHA +ROOT=linkml-validation-reports ``` +Raw metadata is schema-independent and only fetched once; subsequent +runs reuse it. Schema-dependent files (`metadata_migrated.json`, +`validation.{json,txt}`, `SUMMARY.md`, top-level `README.md`) are +rewritten in place when the schema content changes. + ### 1. Fetch metadata ```sh @@ -53,9 +57,10 @@ hatch run linkml-auto-converted:python \ Downloads `metadata.json` + `info.json` for every dandiset's draft and every published version into `$ROOT/data///`. -Re-running is safe — already-downloaded versions are skipped unless -`--refresh` is passed. `--limit N` truncates to N dandisets for smoke -tests. `-i ` selects a non-production DANDI instance. +Already-downloaded versions are skipped. `--refresh` is a forceful +override that re-downloads everything regardless. `--limit N` +truncates to N dandisets for smoke tests. `-i ` selects a +non-production DANDI instance. ### 2. Migrate + validate @@ -69,10 +74,18 @@ For each version directory, runs `dandischema.metadata.migrate` on the raw metadata first, then validates the migrated instance against the LinkML schema (drafts → `Dandiset`, published → `PublishedDandiset`). Writes `metadata_migrated.json` (when migration succeeds), plus -`validation.json` (structured record carrying `migration_status`), -`validation.txt`, and `SUMMARY.md`. Versions whose migration fails -are recorded with the error and skipped for validation. `--refresh` -re-runs migration and validation on already-processed versions. +`validation.json` (structured record carrying `migration_status` and +`schema_sha256`), `validation.txt`, and `SUMMARY.md`. Versions whose +migration fails are recorded with the error and skipped for +validation. + +The resume guard is schema-aware: each `validation.json` is stamped +with the SHA-256 of the schema file's bytes, and a re-run skips a +version only when its stamp matches the current schema. So changing +`dandischema/models.yaml` (committed or uncommitted) automatically +re-runs migration and validation for every version on the next call — +no flag needed. `--refresh` is a forceful override that ignores the +stamp and re-runs everything regardless. ### 3. Generate report @@ -86,7 +99,7 @@ hatch run linkml-auto-converted:python \ Writes `$ROOT/README.md`: overall counts, then per-bucket tables (target class × schemaVersion) with top error patterns and links to -each version's `SUMMARY.md`. +each version's `SUMMARY.md`. Always rewritten on invocation. For details on the on-disk layout, JSON field shapes, and design rationale, read the module docstrings of the three scripts directly. diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py index 3d558653..02a7eb49 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/fetch_metadata.py @@ -27,7 +27,7 @@ ------- :: - python fetch_metadata.py linkml-validation-reports//data + python fetch_metadata.py linkml-validation-reports/data """ from __future__ import annotations diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py index e51941f9..ab386711 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/generate_report.py @@ -3,13 +3,14 @@ The report directory is expected to look like:: - linkml-validation-reports// + linkml-validation-reports/ ├── README.md <-- written by this script └── data/ ├── 000003/ │ ├── draft/ │ │ ├── metadata.json │ │ ├── info.json + │ │ ├── metadata_migrated.json │ │ ├── validation.json │ │ ├── validation.txt │ │ └── SUMMARY.md @@ -42,7 +43,7 @@ ------- :: - python generate_report.py linkml-validation-reports/ \\ + python generate_report.py linkml-validation-reports \\ --commit-hash 54085828c72b69f3b9933dbd288114a9d074ed46 \\ --commit-date 2026-04-20T18:47:47-07:00 """ @@ -306,7 +307,7 @@ def main( report_root: Path = typer.Argument( ..., help="Top-level report directory, i.e. " - "linkml-validation-reports//. " + "linkml-validation-reports/. " "Must contain a `data/` subdirectory of validated versions.", ), commit_hash: str = typer.Option( diff --git a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py index 24fffe54..7ba5f2f7 100644 --- a/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py +++ b/.claude/skills/dandi-linkml-validation-report/scripts/validate_metadata.py @@ -49,20 +49,34 @@ * ``Dandiset`` — for the ``draft`` version * ``PublishedDandiset`` — for any non-``draft`` (i.e. published) version -Re-running is safe: by default versions that already have a -``validation.json`` are skipped. Pass ``--refresh`` to re-run migration -and validation for every version. +Re-running is schema-aware. The script stamps the SHA-256 of the +schema file's bytes into each ``validation.json`` as +``schema_sha256``. On a re-run it skips a version only when the +existing record was produced against the *same* schema content. If +the schema file has changed (committed edit, uncommitted edit, +swapped to a different file — anything that changes the byte +content), migration and validation re-run automatically and the +schema-dependent files are rewritten in place. ``metadata.json`` / +``info.json`` are never touched here; they're owned by +``fetch_metadata.py``. + +``--refresh`` is a forceful override: it ignores the resume guard +and re-runs migration, validation, and the rewrite of every +schema-dependent file regardless of stamp. You shouldn't need it for +normal "I changed the schema, re-validate" workflows — those are +already automatic. Example ------- :: - python validate_metadata.py linkml-validation-reports//data \\ + python validate_metadata.py linkml-validation-reports/data \\ --schema dandischema/models.yaml """ from __future__ import annotations +import hashlib import json import logging import os @@ -141,8 +155,12 @@ def _atomic_write_json(path: Path, data: object) -> None: def _validate_one( - validator: Validator, version_dir: Path, *, refresh: bool -) -> tuple[str, str, int]: + validator: Validator, + version_dir: Path, + *, + schema_sha256: str, + refresh: bool, +) -> None: """Migrate then validate one ``/`` directory. Reads ``info.json`` (written by ``fetch_metadata.py``) to decide the @@ -151,10 +169,14 @@ def _validate_one( LinkML schema. Writes ``metadata_migrated.json`` (on success), ``validation.json`` / ``validation.txt`` / ``SUMMARY.md``. - Returns ``(target_class, migration_status, n_results)`` so the - caller can log a tally without re-reading the file. - ``migration_status`` is ``"success"`` or ``"failed"``; - ``n_results`` is ``0`` when migration failed. + The resume guard skips a version only when its existing + ``validation.json`` was produced against a schema with the same + ``schema_sha256`` we were given. A schema-content change therefore + forces an automatic re-run without ``--refresh``. + + Logs one ``INFO`` line per version describing the outcome + (``resumed``, ``migrated, validated``, or + ``migration failed; validation skipped``). """ metadata_file = version_dir / "metadata.json" info_file = version_dir / "info.json" @@ -168,7 +190,10 @@ def _validate_one( target_class = "PublishedDandiset" if is_published else "Dandiset" # Resume support: if we already have a JSON record for this version - # and the caller didn't pass --refresh, leave the directory alone. + # *produced against the same schema content*, and the caller didn't + # pass --refresh, leave the directory alone. A schema-content + # change makes ``existing["schema_sha256"]`` fail the equality + # check and falls through to a fresh migration + validation. if out_json.exists() and not refresh: try: existing = json.loads(out_json.read_text()) @@ -179,11 +204,15 @@ def _validate_one( version_dir, ) else: - return ( - target_class, - existing.get("migration_status", "success"), - problem_count, - ) + if existing.get("schema_sha256") == schema_sha256: + logger.info( + "%s/%s — resumed from existing record (%s, %d problems)", + version_dir.parent.name, + version_dir.name, + existing.get("migration_status", "success"), + problem_count, + ) + return raw = json.loads(metadata_file.read_text()) # ``@context`` is a JSON-LD framing field that's not part of the @@ -266,6 +295,10 @@ def _validate_one( "is_published": is_published, "target_class": target_class, "schema_version": info.get("schema_version"), + # SHA-256 of the schema file's bytes — drives the resume guard + # on the next run, so a schema-content change re-validates + # automatically without needing ``--refresh``. + "schema_sha256": schema_sha256, "migration_status": migration_status, "migration_error": migration_error, "exit_code": exit_code, @@ -332,7 +365,20 @@ def _validate_one( ] out_md.write_text("\n".join(md_lines) + "\n") - return target_class, migration_status, len(results) + if migration_status == "success": + logger.info( + "%s/%s — migrated, validated as %s (%d problems)", + version_dir.parent.name, + version_dir.name, + target_class, + len(results), + ) + else: + logger.info( + "%s/%s — migration failed; validation skipped", + version_dir.parent.name, + version_dir.name, + ) # --------------------------------------------------------------------------- @@ -355,7 +401,12 @@ def main( refresh: bool = typer.Option( False, "--refresh", - help="Re-migrate and re-validate even when validation.json already exists.", + help=( + "Forceful override: re-migrate and re-validate every version, " + "ignoring the resume guard. Not needed for normal " + "schema-changed-so-revalidate workflows — those already happen " + "automatically when the schema file's content changes." + ), ), log_level: str = typer.Option("INFO", "--log-level", "-l"), ) -> None: @@ -375,6 +426,13 @@ def main( validation_plugins=[JsonschemaValidationPlugin(closed=True)], ) + # SHA-256 of the schema file's bytes — stamped into every + # ``validation.json`` so a future run can tell whether the schema + # has changed since that record was produced. Computed once here so + # we don't re-hash per version. + schema_sha256 = hashlib.sha256(schema.read_bytes()).hexdigest() + logger.info("schema sha256 = %s", schema_sha256) + version_dirs = sorted( p for p in root.glob("*/*") if p.is_dir() and (p / "metadata.json").is_file() ) @@ -382,25 +440,10 @@ def main( for vd in version_dirs: try: - target, mig_status, n = _validate_one(validator, vd, refresh=refresh) + _validate_one(validator, vd, schema_sha256=schema_sha256, refresh=refresh) except Exception as e: # Never let one broken dandiset abort the whole run. logger.exception("processing failed for %s: %s", vd, e) - else: - if mig_status == "success": - logger.info( - "%s/%s — migrated, validated as %s (%d problems)", - vd.parent.name, - vd.name, - target, - n, - ) - else: - logger.info( - "%s/%s — migration failed; validation skipped", - vd.parent.name, - vd.name, - ) if __name__ == "__main__": From cc12ce3699e1b71749113b589fb57ad26fc3f1c6 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 7 May 2026 10:11:00 -0700 Subject: [PATCH 56/90] feat: ensure `title` in JSON schema is generated from `title` meta slot in LinkML --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ac6cbfb9..80c9d265 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ dependencies = [ [tool.hatch.envs.linkml-auto-converted.scripts] 2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/linkml_conversion_tools/sanitize-yaml > dandischema/models.yaml" 2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -E -e 's,[a-z]+COLON,,g' > dandischema/models_linkml.py" -2json = "d=dandischema/models_linkml; rm -rf $d && mkdir -p $d && for t in Dandiset Asset PublishedDandiset PublishedAsset; do gen-json-schema -t $t dandischema/models.yaml >| $d/$t.json; done" +2json = "d=dandischema/models_linkml; rm -rf $d && mkdir -p $d && for t in Dandiset Asset PublishedDandiset PublishedAsset; do gen-json-schema --title-from title -t $t dandischema/models.yaml >| $d/$t.json; done" pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && python tools/pubschemata.py $d && mv $d/*/*json $d/" [tool.setuptools.packages.find] From 08e2d11f1ace01c40be0cb0598dbe8f4e6547bd9 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 7 May 2026 11:59:46 -0700 Subject: [PATCH 57/90] Make linkml_conversion script handle stacked branches and new files - patch -p1 --forward: skip hunks that are already applied (default `Assume -R?` would silently revert them since stdin is the diff). - After the patch loop, inspect captured output: abort on real conflicts ("failed" / "FAILED"), tolerate non-zero exit only when a skip indicator ("previously applied" / "reversed" / "skipping patch") is present. - Delete .rej files left by --forward. - Replace `git commit -a` with `git add -A; git commit` so newly introduced files from patched branches are included in the merge commit. --- tools/linkml_conversion | 42 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/tools/linkml_conversion b/tools/linkml_conversion index 7d9945f5..bb42a56e 100755 --- a/tools/linkml_conversion +++ b/tools/linkml_conversion @@ -41,9 +41,43 @@ for b in ${branches_to_merge[@]}; do echo "Applying branch $b patch with following differences" git diff --stat "$master_mergebase"..."$remote"/$b - git diff "$master_mergebase"..."$remote"/$b | patch -p1 + # --forward: skip hunks that are already applied (e.g. when a branch + # in the queue is stacked on top of an earlier one in the queue and + # therefore re-introduces hunks already landed). Without it, `patch` + # would prompt "Assume -R?" with default `y` and silently revert + # those hunks since stdin is occupied by the diff. + # + # `patch --forward` exits non-zero whenever any hunk is skipped, even + # when every skip is "already applied". To distinguish that benign + # case from a real conflict, capture the output and inspect it: + # - "failed" (Apple patch) / "FAILED" (GNU patch) marks a real + # conflict — abort. + # - A skip indicator ("previously applied" / "reversed" / + # "skipping patch") is required to accept a non-zero exit. + # `set +e` is needed because `set -eu` is on; otherwise the script + # would bail before we can read `$?`. + set +e + out=$(git diff "$master_mergebase"..."$remote"/$b | patch -p1 --forward 2>&1) + status=$? + set -e + echo "$out" + if [ "$status" -ne 0 ]; then + if grep -qi 'failed' <<<"$out"; then + echo "patch FAILED for branch $b — see rejects" >&2 + exit 1 + fi + if ! grep -qiE 'previously applied|reversed|skipping patch' <<<"$out"; then + echo "patch exited $status for branch $b without a recognized skip indicator; aborting" >&2 + exit 1 + fi + fi done +# Clean up `.rej` files left behind by `patch --forward` for hunks it +# ignored as already applied. They are harmless but would otherwise +# clutter the tree and could be picked up by `git add -A` below. +find . -name '*.rej' -delete + hatch run linkml-auto-converted:2linkml git add dandischema/models.yaml @@ -59,4 +93,8 @@ $(dirname $0)/linkml_conversion_fromlinkml # because we have pre-commit.ci doing that behind our back too! pre-commit run --all || : -git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches" -a --no-verify +# Stage everything, including new files introduced by patches from the +# branches above (e.g. files added on `master`). `git commit -a` alone +# would skip untracked files and silently leave them out of the merge. +git add -A +git commit -m "'Merged' with linkml-conversion $ver and ${#branches_to_merge[@]} branches" --no-verify From 23666ae815a924873a557a0d3c40d450af105247 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 13 May 2026 23:23:11 -0700 Subject: [PATCH 58/90] Replace sed post-processing in `2pydantic` with gen-pydantic template override The `2pydantic` hatch script previously stripped LinkML's namespace-prefix munging from enum member names with `sed -E 's,[a-z]+COLON,,g'` over the whole generated file. Replace that with an `enum.py.jinja` override passed via `gen-pydantic --template-dir`, so the substitution happens at the exact place the labels are emitted (using `pv.label.split("COLON") | last`) rather than as blind text replacement after the fact. Verified byte-identical output against the previous `sed`-based pipeline on `dandischema/models.yaml` from `linkml-auto-converted` (all 116 `COLON` occurrences are preceded by a lowercase prefix, so `split("COLON") | last` is equivalent to the `sed` substitution on this schema). Co-Authored-By: Claude Code 2.1.141 / Claude Opus claude-opus-4-7 --- pyproject.toml | 2 +- .../pydantic_templates/enum.py.jinja | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tools/linkml_conversion_tools/pydantic_templates/enum.py.jinja diff --git a/pyproject.toml b/pyproject.toml index 80c9d265..98f864e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ dependencies = [ ] [tool.hatch.envs.linkml-auto-converted.scripts] 2linkml = "pydantic2linkml -l INFO -M dandischema/models_merge.yaml -O dandischema/models_overlay.yaml dandischema.models | sed -e 's,0x[0-9a-ef]*,0xADDRESS,g' -E -e 's,([^[:space:]]+):[0-9]+,\\1:NUMBER,g' | tools/linkml_conversion_tools/sanitize-yaml > dandischema/models.yaml" -2pydantic = "gen-pydantic --black dandischema/models.yaml | sed -E -e 's,[a-z]+COLON,,g' > dandischema/models_linkml.py" +2pydantic = "gen-pydantic --black --template-dir tools/linkml_conversion_tools/pydantic_templates dandischema/models.yaml > dandischema/models_linkml.py" 2json = "d=dandischema/models_linkml; rm -rf $d && mkdir -p $d && for t in Dandiset Asset PublishedDandiset PublishedAsset; do gen-json-schema --title-from title -t $t dandischema/models.yaml >| $d/$t.json; done" pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && python tools/pubschemata.py $d && mv $d/*/*json $d/" diff --git a/tools/linkml_conversion_tools/pydantic_templates/enum.py.jinja b/tools/linkml_conversion_tools/pydantic_templates/enum.py.jinja new file mode 100644 index 00000000..fc6fc383 --- /dev/null +++ b/tools/linkml_conversion_tools/pydantic_templates/enum.py.jinja @@ -0,0 +1,18 @@ +class {{ name }}(str{% if values %}, Enum{% endif %}): +{% if description %} + """ + {{ description }} + """ +{% endif %} +{% if values %} + {% for pv in values.values() %} + {{ pv.label.split("COLON") | last }} = "{{pv.value}}" + {% if pv.description %} + """ + {{ pv.description | indent(width=4) }} + """ + {% endif %} + {% endfor %} +{% else %} + pass +{% endif %} From 71be4ecfe669cf45e94428c98ea36d1b088f6580 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 12 May 2026 14:19:50 -0700 Subject: [PATCH 59/90] Add LinkML behavior tests for `required: False -> True` slot_usage refinement Adds a detached hatch env (`linkml-behavior-test`) and a new test tree (`tests/linkml_behavior/`) asserting that `linkml-validate` honors a `slot_usage` entry that tightens an inherited slot's `required` from `False` to `True` while preserving the slot's other inherited constraints (here, `range`). This is the LinkML behavior the LinkML version of `dandischema` relies on; see issue #405. The env is detached (no dandischema install) with only `linkml` and `pytest` as dependencies, and it does not pin a LinkML version so the tests run against the latest release and surface upstream regressions early. The new directory carries its own `pytest.toml` so pytest's config-file discovery stops there and does not inherit the repo-root `tox.ini` `[pytest]` section, which is tailored to the `dandischema` package test suite. A follow-up will add a GitHub Actions workflow that invokes the hatch env so the same checks run in CI. Co-Authored-By: Claude Code 2.1.139 / Claude Opus 4.7 claude-opus-4-7 --- pyproject.toml | 12 +++ tests/linkml_behavior/pytest.toml | 7 ++ .../bad_type_instance.yaml | 1 + .../missing_name_instance.yaml | 1 + .../required_refinement/schema.yaml | 31 ++++++++ .../required_refinement/test_validate.py | 76 +++++++++++++++++++ .../required_refinement/valid_instance.yaml | 1 + 7 files changed, 129 insertions(+) create mode 100644 tests/linkml_behavior/pytest.toml create mode 100644 tests/linkml_behavior/required_refinement/bad_type_instance.yaml create mode 100644 tests/linkml_behavior/required_refinement/missing_name_instance.yaml create mode 100644 tests/linkml_behavior/required_refinement/schema.yaml create mode 100644 tests/linkml_behavior/required_refinement/test_validate.py create mode 100644 tests/linkml_behavior/required_refinement/valid_instance.yaml diff --git a/pyproject.toml b/pyproject.toml index 98f864e1..d525cc8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,18 @@ dependencies = [ 2json = "d=dandischema/models_linkml; rm -rf $d && mkdir -p $d && for t in Dandiset Asset PublishedDandiset PublishedAsset; do gen-json-schema --title-from title -t $t dandischema/models.yaml >| $d/$t.json; done" pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && python tools/pubschemata.py $d && mv $d/*/*json $d/" +# Detached env for testing LinkML behaviors that may not be reliable +[tool.hatch.envs.linkml-behavior-test] +detached = true +installer = "uv" +dependencies = [ + "linkml", + "pytest", +] +[tool.hatch.envs.linkml-behavior-test.scripts] +# Invoke with: `hatch run linkml-behavior-test:test` +test = "pytest -v {args:tests/linkml_behavior}" + [tool.setuptools.packages.find] namespaces = true include = ["dandischema*"] diff --git a/tests/linkml_behavior/pytest.toml b/tests/linkml_behavior/pytest.toml new file mode 100644 index 00000000..150a8ea8 --- /dev/null +++ b/tests/linkml_behavior/pytest.toml @@ -0,0 +1,7 @@ +# Empty `[pytest]` section so pytest's upward config-file walk stops here +# (https://docs.pytest.org/en/stable/reference/customize.html — first match +# wins, no merging) instead of reaching the repo-root `tox.ini`, whose +# `[pytest]` section is tailored to the `dandischema` package test suite +# (e.g. `--cov=dandischema`, `filterwarnings = error`) and is not applicable +# to this detached LinkML-behavior test env. +[pytest] diff --git a/tests/linkml_behavior/required_refinement/bad_type_instance.yaml b/tests/linkml_behavior/required_refinement/bad_type_instance.yaml new file mode 100644 index 00000000..93a57aec --- /dev/null +++ b/tests/linkml_behavior/required_refinement/bad_type_instance.yaml @@ -0,0 +1 @@ +name: 42 diff --git a/tests/linkml_behavior/required_refinement/missing_name_instance.yaml b/tests/linkml_behavior/required_refinement/missing_name_instance.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/tests/linkml_behavior/required_refinement/missing_name_instance.yaml @@ -0,0 +1 @@ +{} diff --git a/tests/linkml_behavior/required_refinement/schema.yaml b/tests/linkml_behavior/required_refinement/schema.yaml new file mode 100644 index 00000000..24f80457 --- /dev/null +++ b/tests/linkml_behavior/required_refinement/schema.yaml @@ -0,0 +1,31 @@ +id: https://example.org/required-refinement +name: required_refinement +description: >- + Minimal schema exercising the LinkML behavior that the LinkML version of + `dandischema` relies on for `required: False -> True` slot_usage + refinements (see https://github.com/dandi/dandi-schema/issues/405). A + child class tightens an inherited optional slot to required while + inheriting the slot's range constraint from the parent class. +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/ +default_prefix: ex +default_range: string +imports: + - linkml:types + +slots: + name: + required: false + range: string + +classes: + Person: + slots: + - name + + Employee: + is_a: Person + slot_usage: + name: + required: true diff --git a/tests/linkml_behavior/required_refinement/test_validate.py b/tests/linkml_behavior/required_refinement/test_validate.py new file mode 100644 index 00000000..ef15b0aa --- /dev/null +++ b/tests/linkml_behavior/required_refinement/test_validate.py @@ -0,0 +1,76 @@ +""" +Tests that `linkml-validate` honors a `slot_usage` entry that refines an +inherited slot from `required: False` to `required: True` while preserving +the slot's other inherited constraints (here, `range`). This is the +LinkML behavior that the LinkML version of `dandischema` relies on. + +See https://github.com/dandi/dandi-schema/issues/405. +""" + +from __future__ import annotations + +from pathlib import Path +import subprocess + +import pytest + +HERE = Path(__file__).parent +SCHEMA = HERE / "schema.yaml" + + +def _validate(target_class: str, instance: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "linkml-validate", + "--schema", + str(SCHEMA), + "--target-class", + target_class, + str(HERE / instance), + ], + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize( + ("target_class", "instance"), + [ + # Child requires `name`; instance supplies a string value + # matching the inherited `range: string` -> valid. + ("Employee", "valid_instance.yaml"), + # Parent class leaves `name` optional -> a missing-name instance + # is valid. + ("Person", "missing_name_instance.yaml"), + # Parent class leaves `name` optional; a string value satisfies + # the slot's `range: string` -> valid. + ("Person", "valid_instance.yaml"), + ], +) +def test_validation_passes(target_class: str, instance: str) -> None: + result = _validate(target_class, instance) + assert result.returncode == 0, ( + f"expected validation to pass for {target_class} <- {instance}, " + f"got rc={result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +@pytest.mark.parametrize( + ("target_class", "instance"), + [ + # The core assertion of issue #405: the child's `required: True` + # override is honored. + ("Employee", "missing_name_instance.yaml"), + # The inherited `range: string` survives the refinement on the child. + ("Employee", "bad_type_instance.yaml"), + # `range: string` is also enforced on the parent class itself. + ("Person", "bad_type_instance.yaml"), + ], +) +def test_validation_fails(target_class: str, instance: str) -> None: + result = _validate(target_class, instance) + assert result.returncode != 0, ( + f"expected validation to fail for {target_class} <- {instance}, " + f"got rc={result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) diff --git a/tests/linkml_behavior/required_refinement/valid_instance.yaml b/tests/linkml_behavior/required_refinement/valid_instance.yaml new file mode 100644 index 00000000..ccdc1594 --- /dev/null +++ b/tests/linkml_behavior/required_refinement/valid_instance.yaml @@ -0,0 +1 @@ +name: Alice From 762393563724daf1726634fc51486ca9d29e1d0b Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 12 May 2026 23:00:55 -0700 Subject: [PATCH 60/90] Validate against JSON Schema generated from the LinkML schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the `required_refinement` behavior tests to also assert that the JSON Schema produced by running `gen-json-schema --title-from title` against `schema.yaml` (matching the invocation in `pyproject.toml`'s `linkml-auto-converted:2json` script) honors the `required: False -> True` `slot_usage` refinement while preserving the inherited `range` constraint. Validation is performed via the `check-jsonschema` CLI, which has JSON Schema `format` keyword validation enabled by default. Adds a `conftest.py` with two session-scoped fixtures: - `json_schemas` — per-target-class JSON schemas generated once from `schema.yaml`. - `json_instances` — the YAML data instances converted once to JSON. The detached `linkml-behavior-test` hatch env now also depends on `check-jsonschema` and `PyYAML`. Co-Authored-By: Claude Code 2.1.139 / Claude Opus 4.7 claude-opus-4-7 --- pyproject.toml | 2 + .../required_refinement/conftest.py | 64 +++++++++++++ .../test_json_schema_validate.py | 89 +++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 tests/linkml_behavior/required_refinement/conftest.py create mode 100644 tests/linkml_behavior/required_refinement/test_json_schema_validate.py diff --git a/pyproject.toml b/pyproject.toml index d525cc8a..4b89d8c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,8 +86,10 @@ pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && pyth detached = true installer = "uv" dependencies = [ + "check-jsonschema", "linkml", "pytest", + "PyYAML", ] [tool.hatch.envs.linkml-behavior-test.scripts] # Invoke with: `hatch run linkml-behavior-test:test` diff --git a/tests/linkml_behavior/required_refinement/conftest.py b/tests/linkml_behavior/required_refinement/conftest.py new file mode 100644 index 00000000..a7a52be7 --- /dev/null +++ b/tests/linkml_behavior/required_refinement/conftest.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess + +import pytest +import yaml + +HERE = Path(__file__).parent +SCHEMA = HERE / "schema.yaml" + +CLASSES = ("Person", "Employee") +INSTANCES = ( + "valid_instance.yaml", + "missing_name_instance.yaml", + "bad_type_instance.yaml", +) + + +@pytest.fixture(scope="session") +def json_schemas(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Path]: + """ + Per-target-class JSON schemas generated once from ``schema.yaml`` via + ``gen-json-schema --title-from title``, matching the invocation used in + ``pyproject.toml``'s ``linkml-auto-converted:2json`` script. Returns a + mapping from class name to the path of the generated JSON schema. + """ + out_dir = tmp_path_factory.mktemp("json_schemas") + schemas: dict[str, Path] = {} + for cls in CLASSES: + out = out_dir / f"{cls}.json" + with out.open("wb") as f: + subprocess.run( + [ + "gen-json-schema", + "--title-from", + "title", + "-t", + cls, + str(SCHEMA), + ], + stdout=f, + check=True, + ) + schemas[cls] = out + return schemas + + +@pytest.fixture(scope="session") +def json_instances(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Path]: + """ + Data instances in YAML converted once to JSON (suitable for tools like + ``check-jsonschema`` that consume JSON). Returns a mapping from the + original YAML filename to the path of the converted JSON file. + """ + out_dir = tmp_path_factory.mktemp("json_instances") + instances: dict[str, Path] = {} + for name in INSTANCES: + data = yaml.safe_load((HERE / name).read_text()) + out = out_dir / (Path(name).stem + ".json") + out.write_text(json.dumps(data)) + instances[name] = out + return instances diff --git a/tests/linkml_behavior/required_refinement/test_json_schema_validate.py b/tests/linkml_behavior/required_refinement/test_json_schema_validate.py new file mode 100644 index 00000000..7ade1e03 --- /dev/null +++ b/tests/linkml_behavior/required_refinement/test_json_schema_validate.py @@ -0,0 +1,89 @@ +""" +Tests that the JSON schema generated from the LinkML schema via +``gen-json-schema --title-from title`` (matching the invocation used in +``pyproject.toml``'s ``linkml-auto-converted:2json`` script) honors a +``slot_usage`` entry that refines an inherited slot from ``required: False`` +to ``required: True`` while preserving the slot's other inherited +constraints (here, ``range``). This is the LinkML behavior that the LinkML +version of `dandischema` relies on. + +Validation is performed via the ``check-jsonschema`` CLI, which has JSON +Schema ``format`` validation enabled by default (see +https://check-jsonschema.readthedocs.io/en/stable/usage.html — disabled +only via ``--disable-formats``). + +See https://github.com/dandi/dandi-schema/issues/405. +""" + +from __future__ import annotations + +from pathlib import Path +import subprocess + +import pytest + + +def _validate(schema: Path, instance: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "check-jsonschema", + "--schemafile", + str(schema), + str(instance), + ], + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize( + ("target_class", "instance"), + [ + # Child requires `name`; instance supplies a string value + # matching the inherited `range: string` -> valid. + ("Employee", "valid_instance.yaml"), + # Parent class leaves `name` optional -> a missing-name instance + # is valid. + ("Person", "missing_name_instance.yaml"), + # Parent class leaves `name` optional; a string value satisfies + # the slot's `range: string` -> valid. + ("Person", "valid_instance.yaml"), + ], +) +def test_validation_passes( + target_class: str, + instance: str, + json_schemas: dict[str, Path], + json_instances: dict[str, Path], +) -> None: + result = _validate(json_schemas[target_class], json_instances[instance]) + assert result.returncode == 0, ( + f"expected validation to pass for {target_class} <- {instance}, " + f"got rc={result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +@pytest.mark.parametrize( + ("target_class", "instance"), + [ + # The core assertion of issue #405: the child's `required: True` + # override is honored. + ("Employee", "missing_name_instance.yaml"), + # The inherited `range: string` survives the refinement on the child. + ("Employee", "bad_type_instance.yaml"), + # `range: string` is also enforced on the parent class itself. + ("Person", "bad_type_instance.yaml"), + ], +) +def test_validation_fails( + target_class: str, + instance: str, + json_schemas: dict[str, Path], + json_instances: dict[str, Path], +) -> None: + result = _validate(json_schemas[target_class], json_instances[instance]) + assert result.returncode != 0, ( + f"expected validation to fail for {target_class} <- {instance}, " + f"got rc={result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) From 4b2f003f6d5882c592eaa95ce69858f2469524e1 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 13 May 2026 18:38:42 -0700 Subject: [PATCH 61/90] Extract shared parametrize cases into `_cases.py` The `test_validate.py` and `test_json_schema_validate.py` files exercise the same six `(target_class, instance)` cases against different validators. Move the case lists into a shared `_cases.py` so adding or adjusting a case updates both test files at once. Co-Authored-By: Claude Code 2.1.141 / Claude Opus 4.7 claude-opus-4-7 --- .../required_refinement/_cases.py | 30 +++++++++++++++++++ .../test_json_schema_validate.py | 29 ++---------------- .../required_refinement/test_validate.py | 29 ++---------------- 3 files changed, 36 insertions(+), 52 deletions(-) create mode 100644 tests/linkml_behavior/required_refinement/_cases.py diff --git a/tests/linkml_behavior/required_refinement/_cases.py b/tests/linkml_behavior/required_refinement/_cases.py new file mode 100644 index 00000000..6aebe22c --- /dev/null +++ b/tests/linkml_behavior/required_refinement/_cases.py @@ -0,0 +1,30 @@ +""" +Shared `(target_class, instance)` case lists for the validation tests in +this directory. Kept here so adding or adjusting a case updates every +test file that exercises the same schema/instances against a different +validator. +""" + +from __future__ import annotations + +PASSING_CASES: list[tuple[str, str]] = [ + # Child requires `name`; instance supplies a string value + # matching the inherited `range: string` -> valid. + ("Employee", "valid_instance.yaml"), + # Parent class leaves `name` optional -> a missing-name instance + # is valid. + ("Person", "missing_name_instance.yaml"), + # Parent class leaves `name` optional; a string value satisfies + # the slot's `range: string` -> valid. + ("Person", "valid_instance.yaml"), +] + +FAILING_CASES: list[tuple[str, str]] = [ + # The core assertion of issue #405: the child's `required: True` + # override is honored. + ("Employee", "missing_name_instance.yaml"), + # The inherited `range: string` survives the refinement on the child. + ("Employee", "bad_type_instance.yaml"), + # `range: string` is also enforced on the parent class itself. + ("Person", "bad_type_instance.yaml"), +] diff --git a/tests/linkml_behavior/required_refinement/test_json_schema_validate.py b/tests/linkml_behavior/required_refinement/test_json_schema_validate.py index 7ade1e03..9ca957d9 100644 --- a/tests/linkml_behavior/required_refinement/test_json_schema_validate.py +++ b/tests/linkml_behavior/required_refinement/test_json_schema_validate.py @@ -20,6 +20,7 @@ from pathlib import Path import subprocess +from _cases import FAILING_CASES, PASSING_CASES import pytest @@ -37,20 +38,7 @@ def _validate(schema: Path, instance: Path) -> subprocess.CompletedProcess[str]: ) -@pytest.mark.parametrize( - ("target_class", "instance"), - [ - # Child requires `name`; instance supplies a string value - # matching the inherited `range: string` -> valid. - ("Employee", "valid_instance.yaml"), - # Parent class leaves `name` optional -> a missing-name instance - # is valid. - ("Person", "missing_name_instance.yaml"), - # Parent class leaves `name` optional; a string value satisfies - # the slot's `range: string` -> valid. - ("Person", "valid_instance.yaml"), - ], -) +@pytest.mark.parametrize(("target_class", "instance"), PASSING_CASES) def test_validation_passes( target_class: str, instance: str, @@ -64,18 +52,7 @@ def test_validation_passes( ) -@pytest.mark.parametrize( - ("target_class", "instance"), - [ - # The core assertion of issue #405: the child's `required: True` - # override is honored. - ("Employee", "missing_name_instance.yaml"), - # The inherited `range: string` survives the refinement on the child. - ("Employee", "bad_type_instance.yaml"), - # `range: string` is also enforced on the parent class itself. - ("Person", "bad_type_instance.yaml"), - ], -) +@pytest.mark.parametrize(("target_class", "instance"), FAILING_CASES) def test_validation_fails( target_class: str, instance: str, diff --git a/tests/linkml_behavior/required_refinement/test_validate.py b/tests/linkml_behavior/required_refinement/test_validate.py index ef15b0aa..8ca1fd78 100644 --- a/tests/linkml_behavior/required_refinement/test_validate.py +++ b/tests/linkml_behavior/required_refinement/test_validate.py @@ -12,6 +12,7 @@ from pathlib import Path import subprocess +from _cases import FAILING_CASES, PASSING_CASES import pytest HERE = Path(__file__).parent @@ -34,20 +35,7 @@ def _validate(target_class: str, instance: str) -> subprocess.CompletedProcess[s ) -@pytest.mark.parametrize( - ("target_class", "instance"), - [ - # Child requires `name`; instance supplies a string value - # matching the inherited `range: string` -> valid. - ("Employee", "valid_instance.yaml"), - # Parent class leaves `name` optional -> a missing-name instance - # is valid. - ("Person", "missing_name_instance.yaml"), - # Parent class leaves `name` optional; a string value satisfies - # the slot's `range: string` -> valid. - ("Person", "valid_instance.yaml"), - ], -) +@pytest.mark.parametrize(("target_class", "instance"), PASSING_CASES) def test_validation_passes(target_class: str, instance: str) -> None: result = _validate(target_class, instance) assert result.returncode == 0, ( @@ -56,18 +44,7 @@ def test_validation_passes(target_class: str, instance: str) -> None: ) -@pytest.mark.parametrize( - ("target_class", "instance"), - [ - # The core assertion of issue #405: the child's `required: True` - # override is honored. - ("Employee", "missing_name_instance.yaml"), - # The inherited `range: string` survives the refinement on the child. - ("Employee", "bad_type_instance.yaml"), - # `range: string` is also enforced on the parent class itself. - ("Person", "bad_type_instance.yaml"), - ], -) +@pytest.mark.parametrize(("target_class", "instance"), FAILING_CASES) def test_validation_fails(target_class: str, instance: str) -> None: result = _validate(target_class, instance) assert result.returncode != 0, ( From fb7482527d641cfd02d966716809041fa523e698 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 14 May 2026 22:29:14 -0700 Subject: [PATCH 62/90] Validate against Pydantic models generated from the LinkML schema Add `test_pydantic_validate.py`, mirroring `test_validate.py` and `test_json_schema_validate.py`. A new session-scoped `pydantic_module` fixture runs `gen-pydantic --black --template-dir <...>` (matching the invocation in the `linkml-auto-converted:2pydantic` script) and loads the generated module dynamically; the new `instance_data` fixture provides YAML instances parsed into Python dicts for `model_validate`. The dynamically-loaded module is registered in `sys.modules` under a topic-folder-suffixed name so future sibling fixtures don't collide. Also add `black` and `pydantic` to the `linkml-behavior-test` env so `--black` formatting works and `pydantic.ValidationError` is importable. Co-Authored-By: Claude Code 2.1.141 / Claude Opus 4.7 claude-opus-4-7 --- pyproject.toml | 2 + .../required_refinement/conftest.py | 53 +++++++++++++++++++ .../test_pydantic_validate.py | 42 +++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 tests/linkml_behavior/required_refinement/test_pydantic_validate.py diff --git a/pyproject.toml b/pyproject.toml index 4b89d8c5..095674d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,8 +86,10 @@ pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && pyth detached = true installer = "uv" dependencies = [ + "black", # Allows `gen-pydantic --black` to format the generated Pydantic models "check-jsonschema", "linkml", + "pydantic", # For loading the generated Pydantic models in tests "pytest", "PyYAML", ] diff --git a/tests/linkml_behavior/required_refinement/conftest.py b/tests/linkml_behavior/required_refinement/conftest.py index a7a52be7..252a7eb3 100644 --- a/tests/linkml_behavior/required_refinement/conftest.py +++ b/tests/linkml_behavior/required_refinement/conftest.py @@ -1,14 +1,21 @@ from __future__ import annotations +import importlib.util import json from pathlib import Path import subprocess +import sys +from types import ModuleType import pytest import yaml HERE = Path(__file__).parent SCHEMA = HERE / "schema.yaml" +REPO_ROOT = HERE.parents[2] +PYDANTIC_TEMPLATE_DIR = ( + REPO_ROOT / "tools" / "linkml_conversion_tools" / "pydantic_templates" +) CLASSES = ("Person", "Employee") INSTANCES = ( @@ -62,3 +69,49 @@ def json_instances(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Path]: out.write_text(json.dumps(data)) instances[name] = out return instances + + +@pytest.fixture(scope="session") +def pydantic_module(tmp_path_factory: pytest.TempPathFactory) -> ModuleType: + """ + Pydantic models generated once from ``schema.yaml`` via + ``gen-pydantic --black --template-dir <...>``, matching the invocation + used in ``pyproject.toml``'s ``linkml-auto-converted:2pydantic`` + script. The generated module is loaded and returned so tests can look + up classes by name via ``getattr``. + """ + out_dir = tmp_path_factory.mktemp("pydantic") + out = out_dir / "models_linkml.py" + with out.open("wb") as f: + subprocess.run( + [ + "gen-pydantic", + "--black", + "--template-dir", + str(PYDANTIC_TEMPLATE_DIR), + str(SCHEMA), + ], + stdout=f, + check=True, + ) + # Suffix the module name with the topic folder so sibling + # `tests/linkml_behavior//` fixtures don't collide in + # `sys.modules`. + spec = importlib.util.spec_from_file_location(f"models_linkml_{HERE.name}", out) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # Register before `exec_module` so any forward references in the + # generated code resolve via `sys.modules[__name__]` during import. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="session") +def instance_data() -> dict[str, dict]: + """ + Data instances in YAML loaded once into Python dicts (suitable for + Pydantic's ``model_validate``). Returns a mapping from the original + YAML filename to the parsed dict. + """ + return {name: yaml.safe_load((HERE / name).read_text()) for name in INSTANCES} diff --git a/tests/linkml_behavior/required_refinement/test_pydantic_validate.py b/tests/linkml_behavior/required_refinement/test_pydantic_validate.py new file mode 100644 index 00000000..9cfc9a86 --- /dev/null +++ b/tests/linkml_behavior/required_refinement/test_pydantic_validate.py @@ -0,0 +1,42 @@ +""" +Tests that the Pydantic models generated from the LinkML schema via +``gen-pydantic --black --template-dir <...>`` (matching the invocation +used in ``pyproject.toml``'s ``linkml-auto-converted:2pydantic`` script) +honor a ``slot_usage`` entry that refines an inherited slot from +``required: False`` to ``required: True`` while preserving the slot's +other inherited constraints (here, ``range``). This is the LinkML +behavior that the LinkML version of `dandischema` relies on. + +See https://github.com/dandi/dandi-schema/issues/405. +""" + +from __future__ import annotations + +from types import ModuleType + +from _cases import FAILING_CASES, PASSING_CASES +from pydantic import ValidationError +import pytest + + +@pytest.mark.parametrize(("target_class", "instance"), PASSING_CASES) +def test_validation_passes( + target_class: str, + instance: str, + pydantic_module: ModuleType, + instance_data: dict[str, dict], +) -> None: + cls = getattr(pydantic_module, target_class) + cls.model_validate(instance_data[instance]) + + +@pytest.mark.parametrize(("target_class", "instance"), FAILING_CASES) +def test_validation_fails( + target_class: str, + instance: str, + pydantic_module: ModuleType, + instance_data: dict[str, dict], +) -> None: + cls = getattr(pydantic_module, target_class) + with pytest.raises(ValidationError): + cls.model_validate(instance_data[instance]) From 04afaafbfbc7c011e498a16058d6bee1875f495c Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 14 May 2026 22:40:53 -0700 Subject: [PATCH 63/90] build: pin python version of `linkml-behavior-test` env Set version environment to 3.10. The lowest currently supported Python --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 095674d4..ea2de6db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ pydantic2json = "d=dandischema/models_pydantic; rm -rf $d && mkdir -p $d && pyth [tool.hatch.envs.linkml-behavior-test] detached = true installer = "uv" +python = "3.10" dependencies = [ "black", # Allows `gen-pydantic --black` to format the generated Pydantic models "check-jsonschema", From 37e0801bee02c1040c82c8d5f9236fb603270b16 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 14 May 2026 23:08:36 -0700 Subject: [PATCH 64/90] Add GitHub Actions workflow for LinkML behavior tests Runs the `tests/linkml_behavior/` suite under the `linkml-behavior-test` hatch env on push/PR to `master`, on a daily 06:00 UTC schedule, and on manual dispatch. The env doesn't pin a LinkML version, so the daily schedule surfaces upstream LinkML regressions early. Co-Authored-By: Claude Code 2.1.141 / Claude Opus 4.7 claude-opus-4-7 --- .github/workflows/test-linkml-behavior.yml | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/test-linkml-behavior.yml diff --git a/.github/workflows/test-linkml-behavior.yml b/.github/workflows/test-linkml-behavior.yml new file mode 100644 index 00000000..f1767573 --- /dev/null +++ b/.github/workflows/test-linkml-behavior.yml @@ -0,0 +1,26 @@ +name: LinkML behavior tests + +# Runs the LinkML behavior tests under the `linkml-behavior-test` hatch +# env, which does not pin a LinkML version, so the daily schedule +# surfaces upstream LinkML regressions early. +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: '0 6 * * *' + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Install Hatch + uses: pypa/hatch@install + + - name: Run LinkML behavior tests + run: hatch run linkml-behavior-test:test From 8580859554815b677a1e8a4259603aa236806894 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 14 May 2026 23:55:42 -0700 Subject: [PATCH 65/90] Add detached `linkml-behavior-typing` hatch env The env provides `hatch run linkml-behavior-typing:check`, which runs `mypy --install-types --non-interactive` over `tests/linkml_behavior/`. It is kept distinct from `linkml-behavior-test` and from the tox `typing` env (which targets `dandischema`). Co-Authored-By: Claude Code 2.1.141 / Claude Opus 4.7 claude-opus-4-7 --- pyproject.toml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index ea2de6db..9b647f06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,22 @@ dependencies = [ # Invoke with: `hatch run linkml-behavior-test:test` test = "pytest -v {args:tests/linkml_behavior}" +# Detached env for type-checking the LinkML behavior tests; kept +# separate from `linkml-behavior-test` and from the tox `typing` env +# (which targets `dandischema`). +[tool.hatch.envs.linkml-behavior-typing] +detached = true +python = "3.10" +dependencies = [ + "mypy>=1.0.0", + "pydantic", + "pytest", + "types-PyYAML", +] +[tool.hatch.envs.linkml-behavior-typing.scripts] +# Invoke with: `hatch run linkml-behavior-typing:check` +check = "mypy --install-types --non-interactive {args:tests/linkml_behavior}" + [tool.setuptools.packages.find] namespaces = true include = ["dandischema*"] From 7aa10b7c1c195f22d682e809aa671af8eaabcc44 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 14 May 2026 23:56:49 -0700 Subject: [PATCH 66/90] Add GitHub Actions workflow for type-checking the LinkML behavior tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs `hatch run linkml-behavior-typing:check` on push/PR to `master`. No cron — typing breakages from upstream stub or library releases are rare and typically don't cause runtime errors. Co-Authored-By: Claude Code 2.1.141 / Claude Opus 4.7 claude-opus-4-7 --- .github/workflows/typing-linkml-behavior.yml | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/typing-linkml-behavior.yml diff --git a/.github/workflows/typing-linkml-behavior.yml b/.github/workflows/typing-linkml-behavior.yml new file mode 100644 index 00000000..37423f44 --- /dev/null +++ b/.github/workflows/typing-linkml-behavior.yml @@ -0,0 +1,22 @@ +name: Type-check LinkML behavior tests + +# Static type-checking for `tests/linkml_behavior/` under the +# `linkml-behavior-typing` hatch env. +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + type-check: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Install Hatch + uses: pypa/hatch@install + + - name: Type-check LinkML behavior tests + run: hatch run linkml-behavior-typing:check From cd892df195e304d61ec7c68958204ca34de4b421 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 15 May 2026 17:25:10 -0700 Subject: [PATCH 67/90] build(tox): pin pytest `testpaths` to `dandischema` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collection is supposed to be confined to `dandischema/` via the literal positional arg in `commands = pytest -v {posargs} dandischema`. Under tox 4.25 (local) that works as intended. Under tox 4.54 (CI), however, the literal `dandischema` following `{posargs}` is dropped from the invocation, leaving pytest to default to rootdir-based collection. That made it walk into `tests/linkml_behavior/required_refinement/`, whose dependencies (PyYAML, linkml, etc.) live in the dedicated `linkml-behavior-test` hatch env rather than the tox env, and fail collection with `ModuleNotFoundError: No module named 'yaml'`. Set `testpaths = dandischema` in the `[pytest]` section so pytest's default-collection target matches the intended scope of this test suite. With that in place, the literal `dandischema` in the tox command is redundant — and was the source of the tox-version-dependent behavior — so drop it. A path explicitly supplied via `{posargs}` (e.g. `tox -e py -- tests/foo.py`) still overrides `testpaths` and is now honored cleanly instead of being combined with `dandischema`. Co-Authored-By: Claude Code 2.1.143 / Claude Opus claude-opus-4-7 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 84f9d380..6984935b 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ passenv = DATACITE_DEV_PASSWORD NO_ET commands = - pytest -v {posargs} dandischema + pytest -v {posargs} [testenv:lint] skip_install = true @@ -32,6 +32,7 @@ commands = [pytest] addopts = --cov=dandischema --tb=short --durations=10 +testpaths = dandischema filterwarnings = error # From d1b4d82fd1fd44602d00df144696b694200090bf Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 28 May 2026 15:11:36 -0700 Subject: [PATCH 68/90] Add foundation playbook for migration to LinkML Introduce `docs/designs/migration_to_linkml_playbook/` as a living, self-updating playbook for the ongoing migration of `dandischema` from Pydantic-defined models to a LinkML-defined schema as the source of truth. The foundation covers the problem statement and success criteria, the current wiring of `./tools/linkml_conversion` (including the `2linkml` / `2pydantic` / `2json` / `pydantic2json` Hatch scripts and the role of each file under `tools/linkml_conversion_tools/`), a repeatable procedure with parity checks against the dandi-archive frontend, an inventory of the patch queue applied during translation (`master`, `remove-discriminated-unions`), and conventions for the `log.md` / `findings.md` / `tools/` / `context/` subfiles. More content will be added in subsequent commits. Co-Authored-By: Claude Code 2.1.154 / Claude Opus 4.7 --- .../migration_to_linkml_playbook/OVERVIEW.md | 150 ++++++++++++++++++ .../context/README.md | 5 + .../context/patch-queue.md | 45 ++++++ .../migration_to_linkml_playbook/findings.md | 19 +++ .../migration_to_linkml_playbook/log.md | 18 +++ .../tools/README.md | 5 + 6 files changed, 242 insertions(+) create mode 100644 docs/designs/migration_to_linkml_playbook/OVERVIEW.md create mode 100644 docs/designs/migration_to_linkml_playbook/context/README.md create mode 100644 docs/designs/migration_to_linkml_playbook/context/patch-queue.md create mode 100644 docs/designs/migration_to_linkml_playbook/findings.md create mode 100644 docs/designs/migration_to_linkml_playbook/log.md create mode 100644 docs/designs/migration_to_linkml_playbook/tools/README.md diff --git a/docs/designs/migration_to_linkml_playbook/OVERVIEW.md b/docs/designs/migration_to_linkml_playbook/OVERVIEW.md new file mode 100644 index 00000000..941fec23 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/OVERVIEW.md @@ -0,0 +1,150 @@ +# Migration to LinkML — Playbook + +> **Status:** in progress. Active development branches: `linkml-conversion` (+ its patch-queue branches), `linkml-auto-converted` (translation output). +> **Entry point.** Read this file first. It links out to everything else; load deeper files only as a step needs them. +> **This playbook is self-updating.** Anyone working on the migration — human or AI assistant — must keep it current as new facts surface. See [Keeping this playbook current](#keeping-this-playbook-current) for what to update and where. + +## Problem + +Migrate `dandischema` from its current Pydantic-defined schema (`dandischema/models.py`) to a **LinkML-defined** schema, so that LinkML becomes the single source of truth and Pydantic models / JSON Schemas are *generated from* it. + +## Success criteria + +The migration is done when **all three** hold: + +1. **Behavioral parity with current Pydantic models.** Data instances accepted/rejected by today's `dandischema/models.py` are accepted/rejected the same way when validated against the LinkML schema. +2. **Generated Pydantic models are drop-in replacements** for the hand-written ones, usable by: + - this repo (`dandischema`), + - [`dandi/dandi-cli`](https://github.com/dandi/dandi-cli), + - [`dandi/dandi-archive`](https://github.com/dandi/dandi-archive). +3. **Generated JSON Schema** matches what the `Dandiset` Pydantic model currently emits, well enough to: + - validate data instances, and + - drive the dandi-archive frontend (form generation / UI). + +## How the translation is wired today + +The conversion is orchestrated by the shell script **[`tools/linkml_conversion`](../../../tools/linkml_conversion)**, which delegates the actual translation work to Hatch scripts defined in `pyproject.toml`. Mental model: + +- **Sources live on `linkml-conversion`** (and on the patch-queue branches it lists — currently `master` and `remove-discriminated-unions`). This is where you edit: + - the Pydantic source: `dandischema/models.py`, + - the LinkML-side inputs consumed by the translator: `dandischema/models_overlay.yaml` (corrections), `dandischema/models_merge.yaml` (merge inputs), + - the import stub: `dandischema/models_importstab.py` (installed as `models.py` on the output branch — see step 4 below). +- **`./tools/linkml_conversion` runs the translation** and writes the result to the **`linkml-auto-converted`** branch (checkout flips during the script; tree must be clean before running). Order of stages: + 1. Apply the **patch queue** of branches on top of `linkml-conversion` — see [`context/patch-queue.md`](context/patch-queue.md). Order in that list matters. + 2. **Pydantic → LinkML** via `hatch run linkml-auto-converted:2linkml`. Pipeline: + ``` + pydantic2linkml -M models_merge.yaml -O models_overlay.yaml dandischema.models + | sed (scrub memory-address/line-number noise for stable output) + | tools/linkml_conversion_tools/sanitize-yaml + > dandischema/models.yaml + ``` + The translator is [`pydantic2linkml`](https://github.com/dandi/pydantic2linkml); it consumes `models_merge.yaml` (`-M`) and `models_overlay.yaml` (`-O`) directly. The `sed` filter and `sanitize-yaml` are stabilization/cleanup. + 3. **Rename dance:** `git mv dandischema/models.py dandischema/models_orig.py`, then `git mv dandischema/models_importstab.py dandischema/models.py`. On `linkml-auto-converted`, the file at `dandischema/models.py` is the import stub from [`dandischema/models_importstab.py`](../../../dandischema/models_importstab.py) on `linkml-conversion` — read it there for current content. It re-exports the generated models from `models_linkml` and currently also pulls a few constants forward from `models_orig.py`. The stub is an evolving file; how those remaining `models_orig` imports get resolved over time is itself part of the open work. The original Pydantic source is preserved on `linkml-auto-converted` as `models_orig.py`. + 4. **LinkML → downstream artifacts** via `./tools/linkml_conversion_fromlinkml`, which orchestrates three Hatch scripts in order: + - `2pydantic` → runs LinkML's `gen-pydantic --black --template-dir tools/linkml_conversion_tools/pydantic_templates dandischema/models.yaml > dandischema/models_linkml.py`. The Pydantic Jinja templates under `tools/linkml_conversion_tools/pydantic_templates/` are the customization point for the generated Pydantic. + - `2json` → for each target class in `{Dandiset, Asset, PublishedDandiset, PublishedAsset}`, runs `gen-json-schema -t dandischema/models.yaml` → `dandischema/models_linkml/.json`. Files are then lowercased/dashed (`PublishedDandiset.json` → `published-dandiset.json`). + - `pydantic2json` → runs `python tools/pubschemata.py` → `dandischema/models_pydantic/*.json` (JSON Schemas derived from the **original** Pydantic models, kept as the parity baseline). + 5. `pre-commit run --all` (best effort), then a commit. +- **Outputs on `linkml-auto-converted`** (all produced by the pipeline above — do not hand-edit): + - `dandischema/models.yaml` — LinkML schema + - `dandischema/models.py` — the installed import stub + - `dandischema/models_orig.py` — original Pydantic source, preserved for reference and for the constants the stub still imports from it + - `dandischema/models_linkml.py` — generated Pydantic + - `dandischema/models_linkml/*.json` — JSON Schemas derived **from the LinkML schema** (one per target class, lowercased/dashed filenames) + - `dandischema/models_pydantic/*.json` — JSON Schemas derived **from the original Pydantic models** (parity baseline, generated via `tools/pubschemata.py`) +- **`tools/linkml_conversion_tools/`** is a general drawer for any tool convenient to the LinkML migration. It is **not** structurally divided into "pipeline" vs "auxiliary" — files just live here, and some of them happen to be wired into the current pipeline. New migration-related tools belong here; whether they end up wired into the pipeline is a separate decision. As of now: + - `sanitize-yaml` → wired in as the final pipe stage of `2linkml`. Internally a sub-pipeline that runs three Python helpers in order, each as a stdin→stdout filter: + 1. `remove_notes_by_pattern.py` — strips `notes:` entries matching a configured regex set. + 2. `remove_slot_usage_schemakey.py` — strips `schemaKey` entries inside `slot_usage` blocks. + 3. `sort_license_type_permissible_values.py` — sorts `enums.LicenseType.permissible_values` alphabetically for stable output. + + All three are run inside the `linkml-auto-converted` Hatch env. To add a new sanitization step, append another pipe to `sanitize-yaml` — that's its documented extension point ("Add further sanitization steps here as additional pipes"). + - `pydantic_templates/` → wired in; consumed by `gen-pydantic --template-dir` in `2pydantic`. + - `find_schemakey_mismatches.py` → not currently called from anywhere; an on-demand tool that prints Pydantic models where the `schemaKey` default ≠ class name. Available to run by hand when a schemaKey question comes up. + + Verify a tool's current wiring from source before assuming — pipeline membership can change without renaming. +- **LinkML-behavior test envs** (separate from `tox`): `tests/linkml_behavior/` runs under its own Hatch envs: + - `hatch run linkml-behavior-test:test` — runs the behavior tests under `pytest`. + - `hatch run linkml-behavior-typing:check` — runs `mypy` against the tests. + + Both envs are **detached** (don't install `dandischema`); they exist because these tests probe LinkML itself, not our package. +- **LinkML-semantics tests:** `tests/linkml_behavior/` (e.g. `tests/linkml_behavior/required_refinement/`) is **not** a parity harness for our migration. Each subdirectory pins a specific *LinkML upstream behavior* that the generated dandischema LinkML relies on — e.g. `required_refinement/` exercises `required: False -> True` via `slot_usage` to defend against the issue tracked in [#405](https://github.com/dandi/dandi-schema/issues/405). Extend these only when a new LinkML semantic our schema depends on needs a contract test; parity testing of our migration belongs elsewhere (see [Approach](#approach--repeatable-procedure)). + +## Upstream tool we own + +[`dandi/pydantic2linkml`](https://github.com/dandi/pydantic2linkml) — the package doing most of the Pydantic→LinkML translation. Because **we own it**, systematic translation bugs should usually be fixed *there*, not patched downstream in this repo's overlays. See [Suggestions → "Push fixes upstream when possible"](#suggestions-open-leads). + +## Approach — repeatable procedure + +The procedure that's known to work. Follow in order; deviations belong in `log.md`. + +1. **Make changes on `linkml-conversion`** (or the relevant patch-queue branch). Commit. Tree must be clean before step 2. +2. **Run `./tools/linkml_conversion`** from the repo root. It checks out `linkml-auto-converted` and writes the regenerated artifacts there. +3. **Inspect the diff on `linkml-auto-converted`** — both `models.yaml` and the generated `models_linkml.py` / JSON Schema files. Anything unexpected is a finding. +4. **Run the test/contract suites.** These check different things — don't conflate them: + - **Full dandischema test suite:** `tox -e py3`. + - **Lint + types:** `tox -e lint,typing`. + - **LinkML-semantics contract tests** under `tests/linkml_behavior/` (`hatch run linkml-behavior-test:test`; type-check with `hatch run linkml-behavior-typing:check`) — these defend the *LinkML upstream behaviors* our schema relies on, not Pydantic↔LinkML parity. +5. **Run a parity check.** The migration's real success criterion (criterion 3) is that the LinkML-derived JSON Schema can drive `dandi-archive`'s frontend the same way the Pydantic-derived one does. Two complementary checks: + - **Cheap structural diff:** compare the two JSON Schema sets produced on `linkml-auto-converted`: LinkML-derived (`dandischema/models_linkml/*.json`, via `gen-json-schema`) vs. Pydantic-derived baseline (`dandischema/models_pydantic/*.json`, via `tools/pubschemata.py`). Anything beyond expected, explained differences is a finding. + - **End-to-end behavioral check (preferred when feasible):** drive the dandi-archive UI through representative flows with **Playwright MCP** and compare LinkML-derived behavior to the Pydantic-derived baseline. Two variants: + - **Local stack:** launch the dandi-archive backend + frontend locally, point them at the LinkML-derived JSON Schema. Gives full control (you can swap schemas, set breakpoints, edit on the fly). Local checkouts of `dandi-archive`, `dandi-cli`, `dandischema`, and `pydantic2linkml` exist on this machine (see the auto-memory pointer for paths). + - **Live production instance:** compare against the deployed frontend at . Faster to reach for and known-good as a baseline, but you can only *observe* — you can't swap in the LinkML-derived schema there. Useful for snapshotting "what the Pydantic-derived schema actually drives the UI to do" and as a sanity reference; not a place to *test* the LinkML side. +6. **Diagnose any divergence** — between Pydantic-validated and LinkML-validated outcomes on the same instance, between the two JSON Schemas, or in the dandi-archive frontend behavior. Record in `log.md`; promote stable conclusions into `findings.md`. +7. **Decide where to fix:** + - **In `pydantic2linkml`** if the issue is a systematic translation gap (whole class of types/constraints mishandled). + - **In `dandischema/models_overlay.yaml` or `dandischema/models_merge.yaml`** if it's a one-off LinkML-side change that can't reasonably be expressed by improving the translator. Both files are consumed directly by `pydantic2linkml` (`-O` and `-M` respectively); pick the one that fits the kind of change being made (overlay for corrections to translated elements, merge for additions/merges). When in doubt, check `pydantic2linkml`'s docs for the exact semantics rather than guessing. + - **In `dandischema/models.py`** if the Pydantic source itself is the right place (e.g. an under-specified field). + - Default preference: upstream first. +8. **Pin the fix with a test** at the layer that caught it — no fix lands without a test that would have caught it: + - A failure that came from a LinkML semantic our schema relies on → add a contract test under `tests/linkml_behavior/`. + - A failure caught by the structural JSON Schema diff → add the comparison (or a stable subset of it) as a checked-in fixture/test. + - A failure caught only end-to-end in dandi-archive → at minimum, record the reproduction in `log.md` and link it from `findings.md`; consider whether it can be reduced to a unit/contract test at one of the earlier layers. + +## Suggestions (open leads) + +Speculative — graduate into the procedure above once confirmed, or kill into `findings.md` with reasoning. + +- **Push fixes upstream when possible.** If a class of LinkML output is wrong for many Pydantic constructs, fix `pydantic2linkml` rather than carrying growing local overrides (`models_overlay.yaml` for corrections, `models_merge.yaml` for merges) here. +- **Use the `dandi-linkml-validation-report` skill** (already present on the LinkML branches under `.claude/skills/`) as a fitness signal — running the LinkML schema against real archive metadata exposes failure modes that synthetic tests miss. +- **Re-examine the patch queue** ([`context/patch-queue.md`](context/patch-queue.md)) when behavior drifts unexpectedly — order matters, and a stale branch in the list can silently revert hunks. Use that inventory to check whether any entry has met its exit criterion and can be retired. +- **Watch out for** Pydantic v2 features that LinkML can't express natively (custom validators, discriminated unions, `Annotated` metadata). These are the most likely sources of overlay accretion. + +## Out of scope (for now) + +- Renaming or restructuring classes purely for LinkML aesthetics — preserve current public class/field names to keep downstream consumers stable. +- Migrating `dandi-cli` / `dandi-archive` consumers off the generated models before parity is proven here. + +## Keeping this playbook current + +**The playbook is a living document, not a snapshot.** Whenever working on this migration — investigating, fixing, reading code, discovering tools, hitting a wall — keep this directory in sync with what is actually true. Stale instructions are worse than missing ones, because they get followed. + +The triggers below are **illustrative, not exhaustive** — they're common shapes the update need takes, not the full set. Anything that would make this playbook a more accurate or more useful guide for the next attempt qualifies, even if it doesn't fit any bullet here. When in doubt, write it down. + +- **New fact discovered** about how the translation, the schema, the consumers, or the upstream tooling behaves → add to `log.md`; if it's stable and trustworthy, promote into `findings.md` and adjust any affected procedure step in this file. +- **Existing claim contradicted** by what you see in the code, the artifacts, or a test run → correct the claim in place (don't just append a footnote elsewhere) and note the correction in `log.md` so the history of *why it changed* is preserved. +- **New tool, script, command, or technique** found useful (whether in this repo, in `pydantic2linkml`, in LinkML's CLI, or anywhere else) → mention it where it would actually be reached for (procedure step, suggestion list, or as its own helper under `tools/`). +- **Procedure step turns out to be wrong, incomplete, or in the wrong order** → edit the [Approach](#approach--repeatable-procedure) section directly. The procedure is the part future attempts execute most literally; outdated steps cost the most. +- **Patch-queue branch changes** (added, removed, retired, exit criterion met) → update [`context/patch-queue.md`](context/patch-queue.md) in the **same** change as the edit to `tools/linkml_conversion`. +- **An open question gets answered** → remove it from the [Open questions](#open-questions--unknowns) list and fold the answer into the relevant section (or into `findings.md`). +- **A suggestion is confirmed or killed** → move it out of [Suggestions](#suggestions-open-leads) into the procedure (if confirmed) or into `findings.md` with the reasoning that retired it (if killed). +- **The success criteria, scope, or wiring change** → update the top of this file. These shape every downstream decision. + +What this looks like in practice during a working session: when something is learned, the playbook edit is part of the same unit of work as the code change or the investigation, not a separate "cleanup" pass deferred to later (which never happens). A commit that lands a fix without touching the playbook, when the playbook had something wrong or missing about that area, is incomplete. + +For an AI assistant working in this directory: treat playbook updates as a default expectation of the task, not an optional extra. If a session surfaces a fact that would have saved time at the start, that fact belongs in `findings.md` (or wherever it fits) before the session ends. + +## How to use this directory + +- **`log.md`** — append-only, dated. Raw attempts, observations, dead ends, partial wins. Don't over-curate. +- **`findings.md`** — distilled, durable conclusions promoted out of `log.md`. The thing a future attempt reads first. +- **`tools/`** — scripts/probes accumulated across attempts (diff helpers, ad-hoc validators, comparators). Each script should have a one-line header comment naming its purpose. +- **`context/`** — background material that doesn't belong in the procedure: design notes, references, deeper explanations of constraints, links to related discussions. + +When working in a fresh conversation, open this `OVERVIEW.md` first, then pull only the files the current step needs. + +## Open questions / unknowns + +- Which behavior gaps (if any) are *intentional* (deliberate cleanup during migration) vs. unintentional regressions from the Pydantic baseline. +- Whether `dandi-archive`'s frontend relies on JSON Schema *extensions* (`$comment`, custom keywords) that LinkML's `gen-json-schema` doesn't currently emit. +- **Frontend impact of removing discriminated unions** — the active blocker on retiring the `remove-discriminated-unions` patch-queue branch (see [`context/patch-queue.md`](context/patch-queue.md)). diff --git a/docs/designs/migration_to_linkml_playbook/context/README.md b/docs/designs/migration_to_linkml_playbook/context/README.md new file mode 100644 index 00000000..32e816fd --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/README.md @@ -0,0 +1,5 @@ +# context/ + +Background material that doesn't belong in the procedure: design notes, references, deeper explanations of constraints, links to related discussions (issues, PRs, upstream tickets in `pydantic2linkml` / `linkml` / `dandi-cli` / `dandi-archive`). + +Keep one topic per file; name files in kebab-case (e.g. `discriminated-unions.md`, `jsonschema-frontend-requirements.md`). diff --git a/docs/designs/migration_to_linkml_playbook/context/patch-queue.md b/docs/designs/migration_to_linkml_playbook/context/patch-queue.md new file mode 100644 index 00000000..0719d624 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/patch-queue.md @@ -0,0 +1,45 @@ +# Patch queue (`tools/linkml_conversion`) + +Inventory of the branches listed in the `branches_to_merge` array of [`tools/linkml_conversion`](../../../../tools/linkml_conversion). These branches are materially **part of the translation's input**: the script applies them, in order, on top of `linkml-conversion` before writing the regenerated artifacts to `linkml-auto-converted`. Order matters (per the script's comment). + +A patch-queue entry is meant to be **temporary**. Each branch should be retired — by landing it on `master` or by removing its need (e.g. fixing the underlying issue upstream in `pydantic2linkml`) — once its exit criterion is met. This file is what lets us audit "can we drop any of these yet?" instead of treating the script's list as tribal knowledge. + +## Entry template + +``` +### `` + +- **Purpose:** what this branch changes and why it's applied during translation. +- **Rationale:** why the change isn't (or can't yet be) on `master` or fixed upstream. +- **Exit criterion:** the concrete condition under which this branch can be dropped from the queue. +- **Status:** active / blocked-on-X / ready-to-retire. +- **Ordering note:** why it sits where it sits in the list (if non-obvious). +``` + +--- + +## Current queue (in apply order) + +### `master` + +- **Purpose:** keep `linkml-conversion` up to date with `master` during each translation run, so the regenerated artifacts reflect the latest upstream Pydantic sources. +- **Rationale:** `linkml-conversion` is a long-lived branch that diverges from `master`; folding `master` in at translation time avoids carrying a manual rebase burden on `linkml-conversion` itself. +- **Exit criterion:** when `linkml-conversion` is itself merged to `master` (i.e. the migration's source-of-truth flip happens), this entry becomes unnecessary. +- **Status:** active. Expected to remain in the queue until the migration completes. +- **Ordering note:** applied first so subsequent patch branches stack on a current base. + +### `remove-discriminated-unions` + +- **Purpose:** remove the use of Pydantic discriminated unions from `dandischema/models.py` so the Pydantic sources fed to the translator don't contain a construct LinkML can't represent. The branch is a single commit on top of `master`, +6/−21 in `dandischema/models.py` only. +- **Rationale:** LinkML has no faithful equivalent to Pydantic v2 discriminated unions. Removing them on a separate branch lets the translation succeed without committing the removal to `master`, because the downstream impact on the **dandi-archive frontend** of dropping discriminated unions is not yet characterized. +- **Exit criterion:** **either** (a) the impact on dandi-archive is assessed and acceptable, at which point this branch lands on `master` and is dropped from the queue; **or** (b) `pydantic2linkml` gains a way to translate discriminated unions into a LinkML construct with equivalent validation behavior, at which point the removal is no longer needed and this branch is dropped without merging. +- **Status:** active, blocked on assessing the frontend impact (and/or on an upstream `pydantic2linkml` improvement). Track in `log.md` as that investigation progresses. +- **Ordering note:** applied after `master` so it patches the current Pydantic sources rather than a stale snapshot. + +--- + +## When changing the queue + +- Edit `tools/linkml_conversion`'s `branches_to_merge` array, and **update this file in the same change** — the script comment authorizes editing the list, but an undocumented edit makes the queue opaque again. +- A new entry without a written **exit criterion** is a smell: if there's no condition under which the branch can be retired, it's effectively a permanent patch and probably belongs on `master` (or upstream) — not in the queue. +- Removing an entry: note in `findings.md` what was learned that allowed retirement, so the rationale isn't lost. diff --git a/docs/designs/migration_to_linkml_playbook/findings.md b/docs/designs/migration_to_linkml_playbook/findings.md new file mode 100644 index 00000000..cafe9bc7 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/findings.md @@ -0,0 +1,19 @@ +# Findings + +Distilled, durable conclusions promoted out of `log.md`. A future attempt should read this **before** the log — these are the things we trust. + +Each finding should answer: **what is true, how do we know, and what does it imply for the migration?** + +## Format + +``` +### + +- **Evidence:** how we established this (link to log entry / commit / test). +- **Implication:** what this changes about the procedure or where to fix things. +- **Confidence:** high / medium / low — and what would lower it. +``` + +--- + + diff --git a/docs/designs/migration_to_linkml_playbook/log.md b/docs/designs/migration_to_linkml_playbook/log.md new file mode 100644 index 00000000..7314a55e --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/log.md @@ -0,0 +1,18 @@ +# Migration log + +Append-only, dated. Raw observations, attempts, dead ends, partial wins. Don't over-curate — promote durable conclusions to `findings.md` once they're stable. + +## Format + +``` +## YYYY-MM-DD — + +**Context:** what triggered this entry (which step in OVERVIEW.md, which branch). +**Observation / attempt:** what happened. +**Outcome:** what we know now, what's still open. +**Next:** concrete next move, if any. +``` + +--- + + diff --git a/docs/designs/migration_to_linkml_playbook/tools/README.md b/docs/designs/migration_to_linkml_playbook/tools/README.md new file mode 100644 index 00000000..3857810d --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/tools/README.md @@ -0,0 +1,5 @@ +# tools/ + +Scripts and probes accumulated across migration attempts: diff helpers, ad-hoc validators, schema comparators, one-off reproducers. + +**Convention:** each script starts with a one-line header comment naming its purpose, plus a short usage example. If a tool proves durable, consider whether it belongs upstream (in `tools/linkml_conversion_tools/` at the repo root, or in `pydantic2linkml`) rather than living here. From 54fd2797bc67429318a9ed268eac99e12f9f4ee2 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 28 May 2026 23:49:35 -0700 Subject: [PATCH 69/90] Add role profiles to the LinkML migration playbook Introduce `context/roles/` in the playbook, with a mandatory `senior-developer.md` baseline (behavioral habits inherited by every agent and subagent) plus topical stubs for Vue, Django, and LinkML slices of the migration. Role files are loaded into a working agent's context when a session touches the corresponding slice; the same files also serve as spawn-prompt material when a subagent is invoked. Each topical stub names its scope, its explicit not-in-scope handoffs to sibling roles, and curated references for filling in its content (including first-party LinkML AGENTS.md / SKILL.md material in `linkml/linkml`, the LinkML specification at w3id.org, and community subagent collections vetted as starting material). OVERVIEW.md gains a pointer to `context/roles/` from the "How to use this directory" section. Co-Authored-By: Claude Code 2.1.154 / Claude Opus 4.7 --- .../migration_to_linkml_playbook/OVERVIEW.md | 3 +- .../context/roles/README.md | 42 +++++++++++++++ .../context/roles/django.md | 40 ++++++++++++++ .../context/roles/linkml.md | 53 +++++++++++++++++++ .../context/roles/senior-developer.md | 24 +++++++++ .../context/roles/vue.md | 39 ++++++++++++++ 6 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 docs/designs/migration_to_linkml_playbook/context/roles/README.md create mode 100644 docs/designs/migration_to_linkml_playbook/context/roles/django.md create mode 100644 docs/designs/migration_to_linkml_playbook/context/roles/linkml.md create mode 100644 docs/designs/migration_to_linkml_playbook/context/roles/senior-developer.md create mode 100644 docs/designs/migration_to_linkml_playbook/context/roles/vue.md diff --git a/docs/designs/migration_to_linkml_playbook/OVERVIEW.md b/docs/designs/migration_to_linkml_playbook/OVERVIEW.md index 941fec23..600c521e 100644 --- a/docs/designs/migration_to_linkml_playbook/OVERVIEW.md +++ b/docs/designs/migration_to_linkml_playbook/OVERVIEW.md @@ -88,7 +88,7 @@ The procedure that's known to work. Follow in order; deviations belong in `log.m 5. **Run a parity check.** The migration's real success criterion (criterion 3) is that the LinkML-derived JSON Schema can drive `dandi-archive`'s frontend the same way the Pydantic-derived one does. Two complementary checks: - **Cheap structural diff:** compare the two JSON Schema sets produced on `linkml-auto-converted`: LinkML-derived (`dandischema/models_linkml/*.json`, via `gen-json-schema`) vs. Pydantic-derived baseline (`dandischema/models_pydantic/*.json`, via `tools/pubschemata.py`). Anything beyond expected, explained differences is a finding. - **End-to-end behavioral check (preferred when feasible):** drive the dandi-archive UI through representative flows with **Playwright MCP** and compare LinkML-derived behavior to the Pydantic-derived baseline. Two variants: - - **Local stack:** launch the dandi-archive backend + frontend locally, point them at the LinkML-derived JSON Schema. Gives full control (you can swap schemas, set breakpoints, edit on the fly). Local checkouts of `dandi-archive`, `dandi-cli`, `dandischema`, and `pydantic2linkml` exist on this machine (see the auto-memory pointer for paths). + - **Local stack:** launch the dandi-archive backend + frontend locally, point them at the LinkML-derived JSON Schema. Gives full control (you can swap schemas, set breakpoints, edit on the fly). Requires local clones of `dandi-archive`, `dandi-cli`, `dandischema`, and `pydantic2linkml`. - **Live production instance:** compare against the deployed frontend at . Faster to reach for and known-good as a baseline, but you can only *observe* — you can't swap in the LinkML-derived schema there. Useful for snapshotting "what the Pydantic-derived schema actually drives the UI to do" and as a sanity reference; not a place to *test* the LinkML side. 6. **Diagnose any divergence** — between Pydantic-validated and LinkML-validated outcomes on the same instance, between the two JSON Schemas, or in the dandi-archive frontend behavior. Record in `log.md`; promote stable conclusions into `findings.md`. 7. **Decide where to fix:** @@ -140,6 +140,7 @@ For an AI assistant working in this directory: treat playbook updates as a defau - **`findings.md`** — distilled, durable conclusions promoted out of `log.md`. The thing a future attempt reads first. - **`tools/`** — scripts/probes accumulated across attempts (diff helpers, ad-hoc validators, comparators). Each script should have a one-line header comment naming its purpose. - **`context/`** — background material that doesn't belong in the procedure: design notes, references, deeper explanations of constraints, links to related discussions. +- **`context/roles/`** — role profiles a working agent loads when handling a slice of the migration. [`senior-developer.md`](context/roles/senior-developer.md) is a **mandatory baseline** every agent (parent or subagent) inherits; topical roles ([`vue.md`](context/roles/vue.md), [`django.md`](context/roles/django.md), [`linkml.md`](context/roles/linkml.md)) stack on top of it. See [`context/roles/README.md`](context/roles/README.md) for how this preserves coupled reasoning while still permitting subagents when isolation is genuinely beneficial. When working in a fresh conversation, open this `OVERVIEW.md` first, then pull only the files the current step needs. diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/README.md b/docs/designs/migration_to_linkml_playbook/context/roles/README.md new file mode 100644 index 00000000..95d78559 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/roles/README.md @@ -0,0 +1,42 @@ +# Roles + +A "role" is a profile of mindset, expertise, and operating habits that a working agent loads when handling a particular slice of this migration. Roles let an agent acquire the specialization a task needs by loading the relevant role files into its context. The same role files are also usable as spawn-prompt material when a subagent is invoked. + +## How roles are used + +- **Default mode — one agent, multiple roles loaded.** Migration work is deeply coupled: a frontend rendering question can implicate the JSON Schema, the LinkML schema, and the Pydantic source all at once (see [Success criteria](../../OVERVIEW.md#success-criteria), especially criterion 3). When a session touches more than one slice, the working agent loads the relevant role files into its context and stacks them — one mind, multiple specializations. +- **Subagent mode — when isolation is the actual benefit.** Subagents *may* be invoked for sub-tasks where isolation pays off: a noisy cross-repo search via `Explore`, an unbiased review of a migration PR, a verifiably-independent piece that can run in parallel for wall-clock speedup. When a subagent is invoked, **load the role(s) appropriate to its task into the spawn prompt** so it inherits the right specialization — subagents do *not* share the parent's loaded skills. + +These two modes are not in conflict. The default mode handles coupled work (most of the migration); the subagent mode is a release valve for the cases listed in the analysis above. + +## The senior-developer baseline + +[`senior-developer.md`](senior-developer.md) is a **mandatory baseline** for every agent acting in this project — the parent agent *and* every subagent it spawns. It describes operating habits (meticulousness, verifying uncertainty, reading local docs first, etc.) rather than topical expertise. Other role files build on top of it; they do not replace it. + +**Rule:** when spawning a subagent, its spawn prompt must include the contents of `senior-developer.md` (inlined, or fetched at the start of its task) so the subagent operates under the same baseline as the parent. A subagent that has not loaded the baseline is operating off-spec. + +## Roles inventory + +- [`senior-developer.md`](senior-developer.md) — mandatory baseline (operating habits, not topical expertise). Inherited by every agent. +- [`vue.md`](vue.md) — Vue / dandi-archive frontend. +- [`django.md`](django.md) — Django / dandi-archive backend. +- [`linkml.md`](linkml.md) — LinkML schema authoring + the Pydantic↔LinkML translation pipeline. + +## Ecosystem references + +The Claude Code skill/subagent ecosystem is large and uneven; treat external definitions as **starting material to lift selectively**, not gospel. As of authoring: + +- [`anthropics/skills`](https://github.com/anthropics/skills) — Anthropic's official `SKILL.md` examples; the canonical reference for the format and frontmatter. +- [`wshobson/agents`](https://github.com/wshobson/agents) — large production-leaning marketplace (~190 agents / ~155 skills) with multi-harness packaging. +- [`VoltAgent/awesome-claude-code-subagents`](https://github.com/VoltAgent/awesome-claude-code-subagents) — community catalog of ~100 specialized subagents, indexed by category and language. +- [`rohitg00/awesome-claude-code-toolkit`](https://github.com/rohitg00/awesome-claude-code-toolkit) — broader toolkit including agents, skills, commands, hooks. +- [`travisvn/awesome-claude-skills`](https://github.com/travisvn/awesome-claude-skills) — curated awesome-list. + +Quality varies across these. When borrowing, prefer lifting *specific habits or descriptions* over wholesale adoption — and credit the source in the role file's References section. + +## Adding a new role + +- One topical area per file. If you can't summarize the scope in one sentence, it's probably two roles. +- State explicitly what the role is **not** responsible for, so stacking multiple roles doesn't double-cover. +- Keep the expertise inventory tight — every loaded role takes context. Don't add a role just to populate the matrix. +- Role files are subject to the [self-updating rule](../../OVERVIEW.md#keeping-this-playbook-current): when something is learned about how this slice of the system actually behaves, update the role file in the same unit of work. diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/django.md b/docs/designs/migration_to_linkml_playbook/context/roles/django.md new file mode 100644 index 00000000..4eba3629 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/roles/django.md @@ -0,0 +1,40 @@ +# Django (dandi-archive backend role) + +Topical role: the Django backend of [`dandi/dandi-archive`](https://github.com/dandi/dandi-archive). + +Stacks on top of [`senior-developer.md`](senior-developer.md) — load both together. + +## Scope + +- Django models, views, serializers, and migrations that depend on `dandischema`'s generated Pydantic models. +- Server-side validation paths that consume `dandischema` (whether via Pydantic models or JSON Schemas). +- Verifying that the generated Pydantic from the LinkML schema is a drop-in replacement for the hand-written `dandischema.models` in this consumer (success criterion 2 — see [OVERVIEW](../../OVERVIEW.md#success-criteria)). + +## Not in scope + +- Vue frontend → see [`vue.md`](vue.md). +- LinkML schema authoring → see [`linkml.md`](linkml.md). +- `pydantic2linkml` internals → see [`linkml.md`](linkml.md). +- [`dandi/dandi-cli`](https://github.com/dandi/dandi-cli) (the other major Python consumer of generated Pydantic) — its own scope; cover it with the senior-developer baseline or split out a `dandi-cli.md` role file later if it accrues distinct concerns. + +## What this role needs to know + +> TODO — to be filled out. Sketch: +> +> - Where in the backend `dandischema` is imported and what it's used for (validation, serialization, response shaping). +> - The Django/Python version floor of `dandi-archive` and which Pydantic v2 features are in use. +> - Local dev story: how to launch the backend pointing at locally-edited `dandischema`, including with the `linkml-auto-converted`'s `models_linkml.py` swapped in. +> - Database-migration sensitivities — anything that would break if the generated Pydantic differs subtly from the hand-written one. + +## References + +External skill/agent definitions to **lift content from** (review for fit before adopting wholesale — these are community collections of varying quality): + +- [`anthropics/skills`](https://github.com/anthropics/skills) — canonical reference for the `SKILL.md` format and frontmatter conventions. +- [`VoltAgent/awesome-claude-code-subagents` → `django-developer.md`](https://github.com/VoltAgent/awesome-claude-code-subagents/blob/main/categories/02-language-specialists/django-developer.md) — Django 4+ specialist covering REST APIs, async views, ORM optimization, admin patterns. Closest off-the-shelf match. +- [`ammohq/agents`](https://github.com/ammohq/agents) — described as a "Supreme Django + DRF + ORM + Pillow expert"; useful if dandi-archive uses DRF heavily (verify before relying). +- [`wshobson/agents`](https://github.com/wshobson/agents) — broader marketplace; check for backend / Django entries. + +`dandi-archive`-specific references: + +- TODO: link the repo's backend `README.md`, `CONTRIBUTING.md`, or `DEVELOPMENT.md` once a session actually exercises the backend locally. diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md b/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md new file mode 100644 index 00000000..2b3ccfab --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md @@ -0,0 +1,53 @@ +# LinkML (schema authoring + translation pipeline role) + +Topical role: the LinkML schema itself, and the Pydantic↔LinkML translation pipeline. Covers the core of the migration's source-of-truth flip. + +Stacks on top of [`senior-developer.md`](senior-developer.md) — load both together. + +## Scope + +- Authoring and reviewing `dandischema/models.yaml` and its inputs (`dandischema/models_overlay.yaml`, `dandischema/models_merge.yaml`). +- The [`pydantic2linkml`](https://github.com/dandi/pydantic2linkml) translator. Owned by the DANDI team; systematic translation gaps are typically fixed here. +- LinkML generators in the pipeline: `gen-pydantic` (via the `2pydantic` Hatch script) and `gen-json-schema` (via `2json`). +- The Pydantic-template customization point at `tools/linkml_conversion_tools/pydantic_templates/`. +- The contract tests under `tests/linkml_behavior/` — pinning specific LinkML upstream behaviors the generated schema relies on (see [issue #405](https://github.com/dandi/dandi-schema/issues/405) for an example). +- Choosing where a fix lands (translator vs. overlay vs. merge vs. Pydantic source) — see [procedure step 7](../../OVERVIEW.md#approach--repeatable-procedure). + +## Not in scope + +- Vue frontend behavior driven by the generated JSON Schema → see [`vue.md`](vue.md). +- Django backend behavior driven by the generated Pydantic → see [`django.md`](django.md). + +## What this role needs to know + +> TODO — to be filled out. Sketch: +> +> - Pydantic v2 features that have no faithful LinkML equivalent (discriminated unions, custom validators, `Annotated` metadata) and the current workaround for each. +> - The exact semantic distinction between `-M` (`models_merge.yaml`) and `-O` (`models_overlay.yaml`) in `pydantic2linkml`'s CLI — currently described loosely in [OVERVIEW](../../OVERVIEW.md#how-the-translation-is-wired-today) ("corrections" vs. "merges") and worth tightening here against `pydantic2linkml`'s own docs. +> - LinkML schema conventions used by the rest of the dandi ecosystem (uri/CURIE schemes, identifier prefixes, range types) so generated schema fits. +> - When to upstream a fix to `pydantic2linkml` vs. when to patch via overlay/merge — heuristic recap from procedure step 7. + +## References + +**First-party LinkML AI guidance (read these first):** + +- [`linkml/linkml/AGENTS.md`](https://github.com/linkml/linkml/blob/main/AGENTS.md) (symlinked from `CLAUDE.md`) — maintainer-authored "Claude Code Notes for LinkML." Covers the UV-workspace monorepo layout (`linkml` and `linkml-runtime` published from one repo), the mandatory `uv run` prefix, and Best Practices that are directly applicable here: prefer doctests + pytest functional style, never mock tests, never weaken failing tests, avoid try/except masking bugs, fail fast, always use type hints. When working inside `linkml/linkml` (or proposing changes upstream), this file's rules supersede generic instincts. +- [`linkml/linkml/.claude/skills/codecov-coverage/SKILL.md`](https://github.com/linkml/linkml/blob/main/.claude/skills/codecov-coverage/SKILL.md) — a real first-party LinkML `SKILL.md`. Useful as a structural example (frontmatter, `allowed-tools`, "When to Use" section, coverage-decrease rule) when this role file is promoted to a skill. The skill itself is about Codecov, not topical LinkML — but the *form* is exemplary. + +Primary sources: + +- [LinkML specification](https://w3id.org/linkml/specification) — the normative spec; the source of truth for what LinkML means, ahead of any tutorial-style docs. +- [LinkML official docs](https://linkml.io/linkml/) — schema syntax, generators (`gen-pydantic`, `gen-json-schema`), runtime. +- [`linkml/linkml`](https://github.com/linkml/linkml) on GitHub. +- [`linkml/linkml-runtime`](https://github.com/linkml/linkml-runtime). +- [`dandi/pydantic2linkml`](https://github.com/dandi/pydantic2linkml) — the translator we own. README, open issues, and source are the authoritative description of `-M`/`-O` semantics and current translation gaps. + +In-tree reference: + +- [`.claude/skills/dandi-linkml-validation-report/SKILL.md`](https://github.com/dandi/dandi-schema/blob/linkml-auto-converted/.claude/skills/dandi-linkml-validation-report/SKILL.md) on the `linkml-auto-converted` branch — a working `SKILL.md` already tied to this migration. Useful as a local example of how a LinkML-flavored skill is shaped. + +Format reference: + +- [`anthropics/skills`](https://github.com/anthropics/skills) — canonical `SKILL.md` format, if/when this role file is promoted to an actual skill. + +**Not yet checked:** other repos in the [`linkml` GitHub org](https://github.com/linkml) (`schema-automator`, `linkml-model`, `linkml-validator`, `linkml-store`, `linkml-project-cookiecutter`, etc.) may also carry AGENTS.md / `.claude/skills/` content. Worth spot-checking when scope expands beyond the two clones we have locally. diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/senior-developer.md b/docs/designs/migration_to_linkml_playbook/context/roles/senior-developer.md new file mode 100644 index 00000000..9b1b5442 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/roles/senior-developer.md @@ -0,0 +1,24 @@ +# Senior developer (mandatory baseline) + +The baseline every agent acting in this project inherits — the parent agent and every subagent. Other role files (`vue.md`, `django.md`, `linkml.md`, …) stack topical expertise *on top of* this; they do not replace it. If a role file's guidance ever appears to conflict with this baseline, the baseline wins. + +## Operating habits + +- **Meticulous.** Land changes that are correct, complete, and don't leave silent loose ends. Read what's actually there before changing it; verify edits did what you intended; don't declare done until the last open thread is closed or explicitly deferred. +- **Don't assume — verify.** When something is uncertain, prefer a quick test, a `grep`, a script run, a `git log` / `git show`, or reading the actual source over speculation. Confidence is not verification; saying "I think" or "probably" is a cue to go check. +- **Read the local map first.** Before working in any repo, look for `CLAUDE.md`, `AGENTS.md`, `README.md`, `CONTRIBUTING.md`, `DEVELOPMENT.md`, `docs/`, the repo's `pyproject.toml` / `package.json` / `Cargo.toml`, the test configuration, and any visible CI workflows. These are the fastest path to a repo's idioms, build/test commands, and gotchas. For this project specifically: `dandi-schema/CLAUDE.md` plus this playbook (`docs/designs/migration_to_linkml_playbook/OVERVIEW.md`). +- **Surface uncertainty honestly.** Distinguish "I verified X" from "I assumed X" in your output. Don't paper over gaps with confident phrasing. If a step rests on an unchecked assumption, name the assumption. +- **Push back honestly.** If a request or suggestion looks ill-advised, say so up front, with the concrete tradeoff named, *before* executing. Don't soften pushback because the request came from a user, a boss, or another agent. Don't reverse position just because someone disagreed — only if the new argument actually outweighs the original one. +- **Honor the repo's own conventions.** Style, commit-message style, branch naming, PR template, lint/format setup, language-version floor — match what the repo already does rather than imposing personal preferences. American English in code, comments, commits, and prose unless the repo says otherwise. +- **Reversibility-aware.** Local edits, branch creation, and tests are cheap. Pushes (especially force-pushes), comments on issues/PRs, sent messages, schema migrations, destructive git operations (`reset --hard`, `clean -f`, branch deletion), and anything that touches shared infrastructure are not. Confirm with the user before taking the second kind, unless durably pre-authorized. +- **Trace before you cut.** When investigating an obstacle (failing test, weird behavior, unfamiliar file), find the root cause before reaching for a workaround. Don't bypass safety checks (`--no-verify`, `--force`, deleting lock files) as a way to make a symptom go away. +- **Keep the playbook current.** This project's playbook is self-updating (see [Keeping this playbook current](../../OVERVIEW.md#keeping-this-playbook-current)). If you uncover a fact, contradict an existing claim, find a better tool, or answer an open question, update the relevant playbook file in the same unit of work — not as deferred cleanup. + +## When acting as (or spawning) a subagent + +- **As a subagent:** the spawn prompt should include this baseline (or an explicit pointer to it). If neither was provided, request it before proceeding on anything non-trivial. +- **As a parent spawning a subagent:** include this baseline in the spawn prompt. Subagents do not inherit the parent's loaded role files automatically; they only know what the prompt tells them. + +## Adding to this file + +This file is the canonical home for cross-cutting agent behaviors that apply to *every* role. New items should be **behaviors** (how to operate) rather than **knowledge** (what to know) — the latter belongs in topical role files. Keep entries short and concrete; aim for one sentence per habit with at most one sentence of clarification. diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/vue.md b/docs/designs/migration_to_linkml_playbook/context/roles/vue.md new file mode 100644 index 00000000..9cac9074 --- /dev/null +++ b/docs/designs/migration_to_linkml_playbook/context/roles/vue.md @@ -0,0 +1,39 @@ +# Vue (dandi-archive frontend role) + +Topical role: Vue and the JSON-Schema-driven UI in [`dandi/dandi-archive`](https://github.com/dandi/dandi-archive). + +Stacks on top of [`senior-developer.md`](senior-developer.md) — load both together. + +## Scope + +- The Vue components and form-generation machinery that consume `dandischema`'s generated JSON Schemas. +- Verifying that the LinkML-derived JSON Schema drives the frontend the same way the Pydantic-derived one does (success criterion 3 — see [OVERVIEW](../../OVERVIEW.md#success-criteria)). +- Driving the UI via Playwright MCP for the end-to-end parity check in [procedure step 5](../../OVERVIEW.md#approach--repeatable-procedure). + +## Not in scope + +- Django backend → see [`django.md`](django.md). +- LinkML schema authoring → see [`linkml.md`](linkml.md). +- `pydantic2linkml` internals → see [`linkml.md`](linkml.md). + +## What this role needs to know + +> TODO — to be filled out. Sketch of what belongs here once the GitHub-inspiration pass lands: +> +> - Where in `dandi-archive` the JSON Schema is consumed (which components, which composables, which validation hooks). +> - JSON Schema features the form generator relies on (or assumes), especially any extensions that LinkML's `gen-json-schema` does not currently emit (see [Open questions](../../OVERVIEW.md#open-questions--unknowns)). +> - The frontend's local dev story: how to launch backend + frontend, where to point them at an alternate JSON Schema for the parity check. +> - Vue 3 idioms used in `dandi-archive` (Composition API vs. Options API, state management, TypeScript usage, test framework). + +## References + +External skill/agent definitions to **lift content from** (review for fit before adopting wholesale — these are community collections of varying quality): + +- [`anthropics/skills`](https://github.com/anthropics/skills) — canonical reference for the `SKILL.md` format and frontmatter conventions. The [`frontend-design`](https://github.com/anthropics/skills/tree/main/skills/frontend-design) and [`webapp-testing`](https://github.com/anthropics/skills/tree/main/skills/webapp-testing) skills are the most directly relevant here. +- [`VoltAgent/awesome-claude-code-subagents` → `vue-expert.md`](https://github.com/VoltAgent/awesome-claude-code-subagents/blob/main/categories/02-language-specialists/vue-expert.md) — a Vue 3 Composition API + Pinia + Nuxt 3 specialist; a useful starter for "what knobs a Vue skill describes." +- [`rohitg00/awesome-claude-code-toolkit` → `vue-specialist.md`](https://github.com/rohitg00/awesome-claude-code-toolkit/blob/main/agents/language-experts/vue-specialist.md) — alternative phrasing of the same role. +- [`wshobson/agents`](https://github.com/wshobson/agents) — large, production-leaning marketplace (191 agents / 155 skills). Search for frontend / Vue entries. + +`dandi-archive`-specific references: + +- TODO: link the repo's frontend `README.md`, `CONTRIBUTING.md`, or `DEVELOPMENT.md` (whichever exist) once a session actually launches the frontend locally. From 43e4426763dbb59115d4809288ba97afef1bd4ac Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 29 May 2026 00:20:31 -0700 Subject: [PATCH 70/90] Populate topical role profiles from verified sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fill in the "What this role needs to know" sections of `vue.md`, `django.md`, and `linkml.md` from primary sources: the local `dandi-archive` (frontend `package.json`, backend `pyproject.toml`, dev docs) and `pydantic2linkml` (README, CLAUDE.md, source). Each role names its stack landscape, the seam where dandischema crosses into that slice, and a curated lift from upstream community subagent definitions with caveats about version mismatches. Shared findings across roles surfaced: - @koumoul/vjsf is the form generator that consumes the JSON Schema in the dandi-archive frontend — the concrete consumer for criterion 3. - dandi-archive pins dandischema==0.12.1 exact; dandi-cli pins dandischema ~= 0.12.0 — the generated Pydantic must satisfy both. - `pydantic2linkml -M` is implemented via deepmerge.always_merger; the README's "values from the file win on conflict" oversimplifies. The actual per-type rule: dicts deep-merge, lists append, sets union, type-mismatches and scalars override. OVERVIEW.md's wiring and procedure step 7 are corrected to match, and a list-replacement escape hatch is documented (use `-O` for top-level lists, or fix on the Pydantic side). Co-Authored-By: Claude Code 2.1.154 / Claude Opus 4.7 --- .../migration_to_linkml_playbook/OVERVIEW.md | 7 +- .../context/roles/django.md | 69 ++++++++++++++-- .../context/roles/linkml.md | 82 +++++++++++++++++-- .../context/roles/vue.md | 55 +++++++++++-- 4 files changed, 193 insertions(+), 20 deletions(-) diff --git a/docs/designs/migration_to_linkml_playbook/OVERVIEW.md b/docs/designs/migration_to_linkml_playbook/OVERVIEW.md index 600c521e..cee6c488 100644 --- a/docs/designs/migration_to_linkml_playbook/OVERVIEW.md +++ b/docs/designs/migration_to_linkml_playbook/OVERVIEW.md @@ -27,7 +27,7 @@ The conversion is orchestrated by the shell script **[`tools/linkml_conversion`] - **Sources live on `linkml-conversion`** (and on the patch-queue branches it lists — currently `master` and `remove-discriminated-unions`). This is where you edit: - the Pydantic source: `dandischema/models.py`, - - the LinkML-side inputs consumed by the translator: `dandischema/models_overlay.yaml` (corrections), `dandischema/models_merge.yaml` (merge inputs), + - the LinkML-side inputs consumed by the translator: `dandischema/models_merge.yaml` (passed via `-M`: deep merge — dicts merge recursively, lists append, file wins only on scalars and type mismatches), `dandischema/models_overlay.yaml` (passed via `-O`: shallow merge — top-level keys only). See [`context/roles/linkml.md`](context/roles/linkml.md#-m-vs--o-semantics-verified-against-pydantic2linkmls-source--toolspy770809) for the full per-type breakdown. - the import stub: `dandischema/models_importstab.py` (installed as `models.py` on the output branch — see step 4 below). - **`./tools/linkml_conversion` runs the translation** and writes the result to the **`linkml-auto-converted`** branch (checkout flips during the script; tree must be clean before running). Order of stages: 1. Apply the **patch queue** of branches on top of `linkml-conversion` — see [`context/patch-queue.md`](context/patch-queue.md). Order in that list matters. @@ -93,7 +93,10 @@ The procedure that's known to work. Follow in order; deviations belong in `log.m 6. **Diagnose any divergence** — between Pydantic-validated and LinkML-validated outcomes on the same instance, between the two JSON Schemas, or in the dandi-archive frontend behavior. Record in `log.md`; promote stable conclusions into `findings.md`. 7. **Decide where to fix:** - **In `pydantic2linkml`** if the issue is a systematic translation gap (whole class of types/constraints mishandled). - - **In `dandischema/models_overlay.yaml` or `dandischema/models_merge.yaml`** if it's a one-off LinkML-side change that can't reasonably be expressed by improving the translator. Both files are consumed directly by `pydantic2linkml` (`-O` and `-M` respectively); pick the one that fits the kind of change being made (overlay for corrections to translated elements, merge for additions/merges). When in doubt, check `pydantic2linkml`'s docs for the exact semantics rather than guessing. + - **In `dandischema/models_merge.yaml`** (consumed by `pydantic2linkml -M`) — deep merge: dicts merge recursively, lists append, file wins only on scalars and type mismatches. Use to override a scalar nested inside generated structure, or to *add* items to a list (e.g. extra `permissible_values`, extra slots). Cannot replace or reorder list items. + - **In `dandischema/models_overlay.yaml`** (consumed by `pydantic2linkml -O`) — shallow merge of top-level keys. Use to add/replace whole top-level elements (classes, enums, prefixes), or to outright replace a top-level list that `-M` would have appended to. + + See [`context/roles/linkml.md`](context/roles/linkml.md#-m-vs--o-semantics-verified-against-pydantic2linkmls-source--toolspy770809) for the full decision matrix. - **In `dandischema/models.py`** if the Pydantic source itself is the right place (e.g. an under-specified field). - Default preference: upstream first. 8. **Pin the fix with a test** at the layer that caught it — no fix lands without a test that would have caught it: diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/django.md b/docs/designs/migration_to_linkml_playbook/context/roles/django.md index 4eba3629..5e9ee99b 100644 --- a/docs/designs/migration_to_linkml_playbook/context/roles/django.md +++ b/docs/designs/migration_to_linkml_playbook/context/roles/django.md @@ -19,12 +19,69 @@ Stacks on top of [`senior-developer.md`](senior-developer.md) — load both toge ## What this role needs to know -> TODO — to be filled out. Sketch: -> -> - Where in the backend `dandischema` is imported and what it's used for (validation, serialization, response shaping). -> - The Django/Python version floor of `dandi-archive` and which Pydantic v2 features are in use. -> - Local dev story: how to launch the backend pointing at locally-edited `dandischema`, including with the `linkml-auto-converted`'s `models_linkml.py` swapped in. -> - Database-migration sensitivities — anything that would break if the generated Pydantic differs subtly from the hand-written one. +### Stack landscape (verified against `dandi-archive/pyproject.toml`) + +- Package: **`dandiapi`** (top-level), Python **>= 3.13**. +- **Django 5.2.x** (NOT 4.x — most external Django subagent profiles target 4+; verify before lifting patterns). +- **Django REST Framework 3.17.x** + `drf-extensions` + `drf-yasg` (OpenAPI/Swagger docs). +- Auth / permissions: `django-allauth`, `django-oauth-toolkit`, `django-guardian` (object-level perms). +- Filtering / extensions: `django-filter`, `django-extensions`, `django-cors-headers`, `django-environ`. +- **Resonant stack:** `django-resonant-settings`, `django-resonant-utils` — opinionated settings/utility layer; understand it before adding settings or storage code. +- **Celery** with multiple queues (`celery`, `calculate_sha256`, `ingest_zarr_archive`, `manifest-worker`); the backend depends on background workers for non-trivial flows. +- **PostgreSQL** (port 5432 in dev). +- Type-stub support via `django-stubs-ext`. + +### dandischema pinning observation (critical for the migration) + +- `dandi-archive` pins **`dandischema==0.12.1`** (schema version 0.7.0), exact pin — the comment in `pyproject.toml` says: *"Pin dandischema to exact version to make explicit which schema version is being used."* +- The sibling consumer `dandi-cli` pins `dandischema ~= 0.12.0` (compatible-release). +- **Implication for the migration:** bumping `dandischema` to a LinkML-derived release requires a coordinated bump in both consumers, and the generated Pydantic must remain importable and behaviorally equivalent across both pin styles (exact and compatible-release). A subtle API or runtime-validation difference can be invisible until one of these two repos breaks. + +### Where dandischema crosses into the backend + +Files importing `dandischema` (as of inspection): + +- `dandiapi/conftest.py`, `dandiapi/api/tests/factories.py`, `dandiapi/api/tests/fuzzy.py` → test infrastructure (factory generation, fuzzy comparators). +- `dandiapi/api/doi.py` → DOI metadata serialization. +- `dandiapi/api/multipart.py` → upload-related metadata. +- `dandiapi/api/tests/test_*.py` → many tests assert on `dandischema` behavior. +- `dandiapi/zarr/tests/test_ingest_zarr_archive.py` → zarr ingest path. + +These are the files most likely to fail loudly if the generated Pydantic diverges from the hand-written one. The test suite is the cheapest first signal for criterion 2 (drop-in replacement). + +### Local dev story + +From [`DEVELOPMENT.md`](https://github.com/dandi/dandi-archive/blob/master/DEVELOPMENT.md): + +- **VSCode Dev Containers** is the recommended quickstart (`Dev Containers: Reopen in Container`). +- **Docker Compose** is the alternative (`docker compose up`). +- Backend dev loop (inside container/host): + - `./manage.py migrate`, `createcachetable`, `createsuperuser --email …`, `create_dev_dandiset --owner …`. + - Three terminals: `./manage.py runserver_plus 0.0.0.0:8000`, the celery worker, and `cd web && npm run dev`. +- To exercise the parity check: swap the generated `models_linkml.py` Pydantic into where `dandischema.models` is currently imported (or install `dandischema` from a LinkML-converted branch into the dev env) and run the test suite + the local backend against the frontend. + +### Operating notes + +- The repo uses **`uv`** for dependency resolution (`uv.lock` checked in). Prefer `uv` commands over plain `pip` when touching deps. +- Resonant settings layer is non-obvious; before adding a setting, check `django-resonant-settings` to see if there's already a knob. +- Tests live alongside their app (`dandiapi/api/tests/`, `dandiapi/zarr/tests/`); idiom is `pytest-django` style. +- DRF schemas are exposed via `drf-yasg` — keep `swagger.py` in sync if URL routes change. + +### Watch-outs (LinkML-side semantic differences that can surface here) + +- The Pydantic generated by `gen-pydantic` (called from the `2pydantic` Hatch script) may differ in subtle ways from hand-written Pydantic: optional-field semantics, validator placement, `model_config` knobs, alias handling, JSON-mode serialization. Backend factories and serializers are the most likely to expose these. +- Discriminated unions (the `remove-discriminated-unions` patch-queue branch) — if the backend currently relies on `discriminator=` resolution at validation time, removing it changes the parse path. Verify with `dandiapi/api/tests/test_schema.py` and the fuzzy-comparison utilities. + +### Generic Django expertise (lifted selectively from upstream community references) + +Useful background, not dandi-archive-specific. Source: [`VoltAgent/django-developer.md`](https://github.com/VoltAgent/awesome-claude-code-subagents/blob/main/categories/02-language-specialists/django-developer.md). **Caveat:** the VoltAgent profile is written against Django 4+; this repo is Django 5.2. Don't lift patterns about `async` views, signals, or settings management without re-verifying for Django 5. + +- ORM hygiene: `select_related` / `prefetch_related` for N+1 prevention, explicit index design, careful migrations. +- DRF idioms: ViewSets, serializers, permission/throttle classes, pagination, API versioning, OpenAPI doc consistency. +- Security: CSRF/XSS, secure cookies, headers, rate limiting — but check this repo's `settings/` first; resonant-settings likely already configures most of these. +- Async views and ASGI deployment — re-verify behavior in Django 5.2 before adopting. + +When borrowing patterns from external Django subagent definitions, verify against `dandi-archive`'s actual conventions (Django 5, resonant stack, celery topology) before applying. ## References diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md b/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md index 2b3ccfab..297ca1f9 100644 --- a/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md +++ b/docs/designs/migration_to_linkml_playbook/context/roles/linkml.md @@ -20,12 +20,82 @@ Stacks on top of [`senior-developer.md`](senior-developer.md) — load both toge ## What this role needs to know -> TODO — to be filled out. Sketch: -> -> - Pydantic v2 features that have no faithful LinkML equivalent (discriminated unions, custom validators, `Annotated` metadata) and the current workaround for each. -> - The exact semantic distinction between `-M` (`models_merge.yaml`) and `-O` (`models_overlay.yaml`) in `pydantic2linkml`'s CLI — currently described loosely in [OVERVIEW](../../OVERVIEW.md#how-the-translation-is-wired-today) ("corrections" vs. "merges") and worth tightening here against `pydantic2linkml`'s own docs. -> - LinkML schema conventions used by the rest of the dandi ecosystem (uri/CURIE schemes, identifier prefixes, range types) so generated schema fits. -> - When to upstream a fix to `pydantic2linkml` vs. when to patch via overlay/merge — heuristic recap from procedure step 7. +### `-M` vs `-O` semantics (verified against `pydantic2linkml`'s source — `tools.py:770–809`) + +This was an open question in OVERVIEW. The README's one-liner ("values from the file win on conflict") oversimplifies; the implementation is more nuanced: + +- **`-M` / `--merge-file`** = **deep merge** via [`deepmerge.always_merger`](https://deepmerge.readthedocs.io/). Per-type strategy: + - **dict** → **recursive deep merge** (descend; the file does *not* override the whole dict). + - **list** → **append** the file's list to the existing one (the file does *not* override the whole list; items accumulate). + - **set** → **union**. + - **type mismatch** (file's value has a different type than the existing) → **file wins**. + - **scalar** (int, str, bool, None) → **file wins**. + - Result is validated against the LinkML meta schema. +- **`-O` / `--overlay-file`** = **shallow merge** of the YAML file into the generated schema (top-level keys only). Result is validated against the LinkML meta schema. + +In this repo: `dandischema/models_merge.yaml` is the `-M` input, `dandischema/models_overlay.yaml` is the `-O` input. + +Picking between them: + +- **Use `-M` (deep merge)** when overriding a *scalar* deep inside the generated structure (e.g. flipping a `required: false` to `true` on a specific slot) or adding items to a list (e.g. extra `permissible_values` on an enum, extra slots on a class). Items accumulate; nested dicts merge. +- **Use `-O` (shallow merge)** when patching at the top of the document — adding/replacing whole top-level keys like extra classes, slots, enums, or prefixes — and you do *not* want the merge to descend. + +Honest gotcha: because `-M` **appends** lists rather than replacing them, you cannot remove or reorder items via `-M` alone. If you need to *replace* a list outright, either switch to `-O` (if the list is top-level) or change the source on the Pydantic side. When in doubt, run the conversion with a minimal patch and diff the resulting `models.yaml` — cheaper than guessing. + +### How `pydantic2linkml` actually works + +From `pydantic2linkml/CLAUDE.md`: it translates Pydantic v2 models by **introspecting Pydantic's internal `core_schema` objects**, not the higher-level model API. Two consequences: + +- The translation is sensitive to **how Pydantic v2 builds its `core_schema`** for a given construct. Changes to `dandischema.models` that look semantically identical may yield different `core_schema` shapes and therefore different LinkML output. +- The translation can be sensitive to **the Pydantic version itself**. `pydantic2linkml`'s test matrix runs Python 3.10–3.13 and includes `dandischema` and `aind-data-schema` as known consumers. Bumping Pydantic upstream may require a matching `pydantic2linkml` adjustment before the dandischema migration sees green again. + +### `pydantic2linkml` conventions (lifted from its `CLAUDE.md`) + +When upstreaming a fix: + +- **Hatch is the env manager.** Test invocations: `hatch run test.py3.10:pytest tests/`, type-check with `hatch run types:check`, lint/format with `ruff check . && ruff format .`, spell-check with `codespell`. +- **Document-currency rule** (mirrors this playbook's self-updating rule): "Whenever you notice that any documentation — `CLAUDE.md`, `README.md`, or any other docs — is outdated or incorrect, update it immediately." +- **Prose wraps at ~79 characters** in docs (code blocks and long URLs exempt). Match this when editing markdown there. + +### LinkML upstream conventions (lifted from `linkml/linkml/AGENTS.md`) + +When working in or proposing changes to `linkml/linkml`: + +- **`uv run`** prefix on every command (UV-workspace monorepo publishing both `linkml` and `linkml-runtime`). +- **Pytest functional style, never `unittest`-OO style.** Modern idioms: `@pytest.mark.parametrize` for combinations. +- **Doctests are first-class** — both explanatory examples and unit tests. For longer cases, write pytest tests. +- **Never mock** unless explicitly requested. *"I need to rely on tests to know if something breaks."* +- **Never weaken a failing test** to make it pass — try harder or ask. +- **Avoid `try/except` that masks bugs.** Fail fast. +- **Always use type hints; always document methods and classes.** +- For tests with external dependencies, use the `integration` pytest mark. + +### Pydantic-v2 features with no faithful LinkML equivalent (current state) + +These are the recurring trouble spots: + +- **Discriminated unions** — no LinkML equivalent; currently worked around by the `remove-discriminated-unions` patch-queue branch (see [`patch-queue.md`](../patch-queue.md)). Exit criterion is either an upstream `pydantic2linkml` improvement or an acceptable assessment of the frontend impact (see [`vue.md`](vue.md)). +- **Custom `@field_validator` / `@model_validator` decorators** — runtime-only behavior; LinkML's static schema can't carry the validator code itself. Watch for accuracy gaps where Pydantic accepts/rejects something the JSON Schema doesn't. +- **`Annotated[..., FieldInfo(...)]` and rich `Field()` metadata** — depending on which metadata is set, the translation may or may not faithfully round-trip. Verify by re-generating Pydantic from the LinkML and diffing against the source. +- **`model_config` knobs** (e.g. `populate_by_name`, JSON-serialization aliases) — easy to forget that these change Pydantic behavior in ways LinkML can't represent declaratively. + +### The two downstream consumers' Pydantic pinning + +- `dandi-archive` pins `dandischema==0.12.1` (exact; schema version 0.7.0). +- `dandi-cli` pins `dandischema ~= 0.12.0` (compatible-release). +- The generated Pydantic must remain importable and behaviorally equivalent for both. See [`django.md`](django.md) for the backend's import sites and [the migration's success criteria](../../OVERVIEW.md#success-criteria). + +### Decision matrix for "where to fix" (refines [procedure step 7](../../OVERVIEW.md#approach--repeatable-procedure)) + +| Symptom | First place to look | Likely fix layer | +|---|---|---| +| Whole class of Pydantic constructs translates wrong | `pydantic2linkml`'s `core_schema` introspection | **Upstream `pydantic2linkml`** | +| One generated class/slot has a specific wrong value | Compare `models.yaml` against `models.py` for that class | **`models_merge.yaml`** (`-M`, file-wins deep merge) | +| Need to add a top-level element (class, enum, prefix) not in the source | Inspect generated `models.yaml` structure | **`models_overlay.yaml`** (`-O`, shallow merge) | +| The Pydantic source itself is under-specified or wrong | `dandischema/models.py` | **`dandischema/models.py`** on `linkml-conversion` | +| Generated Pydantic differs subtly from intent | `tools/linkml_conversion_tools/pydantic_templates/` | **Customize the Jinja templates** consumed by `gen-pydantic` in `2pydantic` | + +Default preference, in order: upstream `pydantic2linkml` > source Pydantic > merge/overlay > template customization. Reach for the lower-leverage tool only when the higher-leverage one can't cleanly express the change. ## References diff --git a/docs/designs/migration_to_linkml_playbook/context/roles/vue.md b/docs/designs/migration_to_linkml_playbook/context/roles/vue.md index 9cac9074..7ca4bd8c 100644 --- a/docs/designs/migration_to_linkml_playbook/context/roles/vue.md +++ b/docs/designs/migration_to_linkml_playbook/context/roles/vue.md @@ -18,12 +18,55 @@ Stacks on top of [`senior-developer.md`](senior-developer.md) — load both toge ## What this role needs to know -> TODO — to be filled out. Sketch of what belongs here once the GitHub-inspiration pass lands: -> -> - Where in `dandi-archive` the JSON Schema is consumed (which components, which composables, which validation hooks). -> - JSON Schema features the form generator relies on (or assumes), especially any extensions that LinkML's `gen-json-schema` does not currently emit (see [Open questions](../../OVERVIEW.md#open-questions--unknowns)). -> - The frontend's local dev story: how to launch backend + frontend, where to point them at an alternate JSON Schema for the parity check. -> - Vue 3 idioms used in `dandi-archive` (Composition API vs. Options API, state management, TypeScript usage, test framework). +### Stack landscape (verified against `dandi-archive/web/package.json`) + +- **Vue 3.5.x** with the Composition API. No Nuxt — the app is a plain Vue SPA built with Vite. +- **UI:** Vuetify 3 (Material Design); `eslint-plugin-vuetify` enforces a few Vuetify-specific rules. +- **State:** Pinia 3. +- **Router:** `vue-router` 4, with `unplugin-vue-router` for file-based route inference. +- **Build:** Vite with `vite-plugin-node-polyfills`; TypeScript via `vue-tsc`. `tsconfig.json` extends `@tsconfig/node24`. +- **Linting:** ESLint flat config (`eslint.config.js`), `eslint-plugin-vue`, `@vue/eslint-config-typescript`. +- **Error tracking:** `@sentry/vue` is wired in — be aware that errors surface to Sentry in non-dev envs. +- **Misc:** `lodash`, `moment` (legacy date handling), `marked` + `dompurify` (rendered Markdown), `axios` for HTTP. + +### The JSON-Schema → UI seam (this is criterion 3's heart) + +This is the part of the frontend the migration must not break: + +- **`@koumoul/vjsf`** (Vue JSON-Schema Form) is the form generator that consumes the JSON Schema and renders the metadata editor. Anything the LinkML-derived JSON Schema fails to support that the Pydantic-derived one supports surfaces here first. +- **`@apidevtools/json-schema-ref-parser`** resolves `$ref`s in the schema before `vjsf` sees the result. Differences in how `gen-json-schema` emits refs (e.g. `$ref` style, `$defs` location) can interact with this stage rather than with `vjsf` itself. +- **TypeScript typings from JSON Schema:** the repo's `web/src/types/schema.ts` is generated by `json-schema-to-typescript` (devDep), invoked via `npm run migrate ` (see [`web/README.md`'s Schema Migration section](https://github.com/dandi/dandi-archive/blob/master/web/README.md)). The typings are *lint-only*; they don't drive runtime behavior. Still useful as a parity probe — diffing the regenerated `schema.ts` between Pydantic-derived and LinkML-derived JSON Schemas is a quick structural signal. +- **The form-driven UI lives in `web/src/components/Meditor/`** — the metadata editor. That's the directory to load in the browser when doing the [end-to-end parity check](../../OVERVIEW.md#approach--repeatable-procedure). + +### Local dev story + +- Frontend dev server: `cd web && npm install && npm run dev` → http://localhost:8085/. +- Backend at http://localhost:8000/ (see [`django.md`](django.md) for how to launch it). The frontend assumes the backend is reachable; without it the UI loads but most flows fail. +- To exercise the parity check: launch the local stack, regenerate the JSON Schema from the LinkML side, then either (a) point the frontend at the LinkML-derived JSON via the backend's schema-serving endpoint (preferred), or (b) drop the LinkML-derived JSON into where `dandischema` would deliver it for that schema version. Drive the Meditor with Playwright MCP and compare to the Pydantic-derived baseline. + +### Operating notes + +- Composition API and `