Sec 41

CHANGELOG.md

@@ -6,6 +6,10 @@ This project adheres to Semantic Versioning and follows a Keep a Changelog-like
 
 ## [Unreleased]
 
+### Changed
+
+-   Full TOON Spec 4.1 conformance: canonical number formatting, BOM stripping, comment pre-pass (§5.1), strict header validation (§5, §6, §7.3, §7.4), nested field groups in tabular arrays (§9.3), and keyed tabular form for objects of uniform objects, including the keyless root form and keyed headers on list-item hyphen lines (§9.5, §10). Conformance suite: 95/95 passing.
+
 ## [2.0.1] - 2026-07-11
 
 ### Added

README.md

@@ -4,7 +4,7 @@
 [![Release](https://github.com/toon-format/toon-java/actions/workflows/release.yml/badge.svg)](https://github.com/toon-format/toon-java/actions/workflows/release.yml)
 [![Maven Central](https://img.shields.io/maven-central/v/dev.toonformat/jtoon.svg)](https://central.sonatype.com/artifact/dev.toonformat/jtoon)
 ![Coverage](.github/badges/jacoco.svg)
-[![SPEC v3.3.2](https://img.shields.io/badge/spec-v3.3.2-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
+[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
 [![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE)
 
 Compact, human-readable serialization format for LLM contexts with **30-60% token reduction** vs JSON. Combines YAML-like indentation with CSV-like tabular arrays. Working towards full compatibility with the [official TOON specification](https://github.com/toon-format/spec).

build.gradle

@@ -149,7 +149,7 @@ dependencies {
     compileOnly 'com.github.spotbugs:spotbugs-annotations:4.10.3'
 
     // NullAway + Error Prone for compile-time null safety
-    errorprone 'com.uber.nullaway:nullaway:0.13.7'
+    errorprone 'com.uber.nullaway:nullaway:0.13.8'
     // Pin error_prone_core to 2.42.0 (Java 17 compatible; 2.50.0+ requires Java 21)
     errorprone('com.google.errorprone:error_prone_core:2.42.0') {
         version {

gradle/verification-metadata.xml

@@ -736,15 +736,15 @@
             <sha256 value="1d8518e3ac7532a104e4f7be77def37c982e530723c6bdb3d67708cce2b0c2c4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.uber.nullaway" name="nullaway" version="0.13.7">
-         <artifact name="nullaway-0.13.7.jar">
-            <sha256 value="6bcb696ddcfa0f7cc12b90face8e7af041063f278cd9f3f07de2c53262e24d68" origin="Generated by Gradle"/>
+      <component group="com.uber.nullaway" name="nullaway" version="0.13.8">
+         <artifact name="nullaway-0.13.8.jar">
+            <sha256 value="c0dd630589f61ac674d212c73e7f60d36f45c302342f95e5f631c64a018fa0d7" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="nullaway-0.13.7.module">
-            <sha256 value="597eb739291a0c429423f51ada02af2b1bd272b38441e4dd9685acecd7f3cb60" origin="Generated by Gradle"/>
+         <artifact name="nullaway-0.13.8.module">
+            <sha256 value="fb4caeed4cd23464d0aaa29cf1cbef0f73702a63067d0c276e03eb2a782447d2" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="nullaway-0.13.7.pom">
-            <sha256 value="29aa9dad25fd0f80b6e5352df90156c5454a65d62efdb77a0638972478c3f862" origin="Generated by Gradle"/>
+         <artifact name="nullaway-0.13.8.pom">
+            <sha256 value="243310e80d3fc532a3430171b43ea4c46ff1fdc951c203ce909d76c8d4fdd692" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="commons-beanutils" name="commons-beanutils" version="1.9.4">

pmd-rules.xml

@@ -61,6 +61,13 @@
         <exclude name="UseUtilityClass" />
     </rule>
 
+    <!-- Complexity gates: keep methods readable; Sonar-compatible threshold -->
+    <rule ref="category/java/design.xml/CognitiveComplexity">
+        <properties>
+            <property name="reportLevel" value="15" />
+        </properties>
+    </rule>
+
     <!-- Error Prone -->
     <rule ref="category/java/errorprone.xml">
         <exclude name="AvoidCatchingGenericException" />

src/main/java/dev/toonformat/jtoon/decoder/ArrayDecoder.java

@@ -9,6 +9,7 @@
 import static dev.toonformat.jtoon.util.Constants.BACKSLASH;
 import static dev.toonformat.jtoon.util.Constants.COLON;
 import static dev.toonformat.jtoon.util.Constants.DOUBLE_QUOTE;
+import static dev.toonformat.jtoon.util.Constants.LIST_ITEM_MARKER;
 import static dev.toonformat.jtoon.util.Constants.LIST_ITEM_PREFIX;
 import static dev.toonformat.jtoon.util.Headers.ARRAY_HEADER_PATTERN;
 import static dev.toonformat.jtoon.util.Headers.TABULAR_HEADER_PATTERN;
@@ -19,11 +20,43 @@
 public final class ArrayDecoder {
 
     private static final int DELIMITER_GROUP_INDEX = 3;
+    private static final int FIELDS_GROUP_INDEX = 4;
 
     private ArrayDecoder() {
         throw new UnsupportedOperationException("Utility class cannot be instantiated");
     }
 
+    /**
+     * Spec §6: the delimiter declared inside the bracket segment of a tabular
+     * header must match the delimiter used by the brace field list. A header
+     * that declares a delimiter the field list does not use is defective.
+     *
+     * @param arrayHeader the array header starting with the bracket segment
+     * @return true when the header carries a mismatched delimiter declaration
+     */
+    static boolean hasTabularDelimiterMismatch(final String arrayHeader) {
+        final Matcher matcher = TABULAR_HEADER_PATTERN.matcher(arrayHeader);
+        if (!matcher.find() || matcher.group(DELIMITER_GROUP_INDEX) == null) {
+            return false;
+        }
+        final char declared = matcher.group(DELIMITER_GROUP_INDEX).charAt(0);
+        boolean inQuotes = false;
+        boolean escaped = false;
+        for (int i = 0; i < matcher.group(FIELDS_GROUP_INDEX).length(); i++) {
+            final char c = matcher.group(FIELDS_GROUP_INDEX).charAt(i);
+            if (escaped) {
+                escaped = false;
+            } else if (c == '\\') {
+                escaped = true;
+            } else if (c == '"') {
+                inQuotes = !inQuotes;
+            } else if (!inQuotes && c != declared && (c == ',' || c == '\t' || c == '|')) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Parses array from the header string and the following lines.
      * Detects array type (tabular, list, or primitive) and routes accordingly.
@@ -85,74 +118,145 @@ static List<Object> parseArrayWithDelimiter(final String header, final int depth
         }
 
         if (arrayMatcher.find()) {
-            // In strict mode, reject bracket lengths with leading zeros (e.g. [03])
-            // unless the length is exactly "0".
-            if (context.options.strict()) {
-                final String lengthStr = arrayMatcher.group(2);
-                if (lengthStr.length() > 1 && lengthStr.charAt(0) == '0') {
-                    throw new IllegalArgumentException(
-                        "Invalid array length with leading zeros: [" + lengthStr + "]");
-                }
-            }
+            rejectLeadingZeroLength(arrayMatcher, context.options.strict());
             final int headerEndIdx = arrayMatcher.end();
             final String afterHeader = header.substring(headerEndIdx).trim();
 
-            if (afterHeader.startsWith(COLON)) {
-                final String inlineContent = afterHeader.substring(1).trim();
-
-                if (!inlineContent.isEmpty()) {
-                    final List<Object> result = parseArrayValues(inlineContent, arrayDelimiter,
-                        context.options.maxArraySize(), context.options.maxStringLength());
-                    validateArrayLength(header, result.size(), context.options.maxArraySize());
-                    context.currentLine++;
-                    return Collections.unmodifiableList(result);
-                }
+            if (hasInlineContent(afterHeader)) {
+                return parseInlineArray(afterHeader, header, arrayDelimiter, context);
             }
 
-            context.currentLine++;
-            if (context.currentLine < context.lines.length) {
-                final String nextLine = context.lines[context.currentLine];
-                final int nextDepth = DecodeHelper.getDepth(nextLine, context);
-                final String nextContent = nextLine.substring(nextDepth * context.options.indent());
-
-                if (nextDepth <= depth) {
-                    // The next line is not a child of this array,
-                    // the array is empty
-                    validateArrayLength(header, 0, context.options.maxArraySize());
-                    return Collections.emptyList();
-                }
+            // Spec §12: blank lines between the header and the first item are
+            // accepted even in strict mode
+            skipBlankLines(context);
 
-                if (nextContent.startsWith(LIST_ITEM_PREFIX)) {
-                    context.currentLine--;
-                    return Collections.unmodifiableList(parseListArray(depth, header, context));
-                } else {
-                    context.currentLine++;
-                    final List<Object> result = parseArrayValues(nextContent, arrayDelimiter,
-                        context.options.maxArraySize(), context.options.maxStringLength());
-                    validateArrayLength(header, result.size(), context.options.maxArraySize());
-                    return Collections.unmodifiableList(result);
-                }
+            if (context.currentLine < context.lines.length) {
+                return parseArrayDataLine(header, depth, arrayDelimiter, context);
             }
-            final List<Object> empty = new ArrayList<>();
-            validateArrayLength(header, 0, context.options.maxArraySize());
-            return Collections.unmodifiableList(empty);
+            validateArrayLength(header, 0, context.options.maxArraySize(), context.options.strict());
+            return Collections.unmodifiableList(new ArrayList<>());
+        }
+
+        // Spec §9.1/§9.2: a bare bracket pair is an empty array header
+        if ("[]".equals(header.trim())) {
+            context.currentLine++;
+            return Collections.emptyList();
         }
 
         if (context.options.strict()) {
             throw new IllegalArgumentException("Invalid array header: " + header);
         }
+        context.currentLine++;
         return Collections.emptyList();
     }
 
+    /**
+     * In strict mode, rejects bracket lengths with leading zeros (e.g. [03])
+     * unless the length is exactly "0".
+     *
+     * @param arrayMatcher the matched array header
+     * @param strict       strict mode flag
+     */
+    private static void rejectLeadingZeroLength(final Matcher arrayMatcher, final boolean strict) {
+        if (strict) {
+            final String lengthStr = arrayMatcher.group(2);
+            if (lengthStr.length() > 1 && lengthStr.charAt(0) == '0') {
+                throw new IllegalArgumentException(
+                    "Invalid array length with leading zeros: [" + lengthStr + "]");
+            }
+        }
+    }
+
+    /**
+     * Returns whether the header text past the bracket segment declares
+     * non-empty inline values ({@code header: v1,v2}).
+     *
+     * @param afterHeader the header text past the bracket segment
+     * @return true when inline values follow the colon
+     */
+    private static boolean hasInlineContent(final String afterHeader) {
+        return afterHeader.startsWith(COLON) && !afterHeader.substring(1).isBlank();
+    }
+
+    /**
+     * Parses the inline values of an array header ({@code header: v1,v2}).
+     *
+     * @param afterHeader    the header text past the bracket segment
+     * @param header         the full header string
+     * @param arrayDelimiter array delimiter
+     * @param context        decode context
+     * @return the parsed values
+     */
+    private static List<Object> parseInlineArray(final String afterHeader, final String header,
+            final Delimiter arrayDelimiter, final DecodeContext context) {
+        final String inlineContent = afterHeader.substring(1).trim();
+        final List<Object> result = parseArrayValues(inlineContent, arrayDelimiter,
+            context.options.maxArraySize(), context.options.maxStringLength());
+        validateArrayLength(header, result.size(), context.options.maxArraySize(), context.options.strict());
+        context.currentLine++;
+        return Collections.unmodifiableList(result);
+    }
+
+    /**
+     * Advances the current line past blank lines following the header.
+     *
+     * @param context decode context
+     */
+    private static void skipBlankLines(final DecodeContext context) {
+        do {
+            context.currentLine++;
+        } while (context.currentLine < context.lines.length
+            && DecodeHelper.isBlankLine(context.lines[context.currentLine]));
+    }
+
+    /**
+     * Parses the first data line below an array header, routing list items to
+     * the list parser and any other content to the value splitter.
+     *
+     * @param header         the full header string
+     * @param depth          depth of the array
+     * @param arrayDelimiter array delimiter
+     * @param context        decode context
+     * @return the parsed array values
+     */
+    private static List<Object> parseArrayDataLine(final String header, final int depth,
+            final Delimiter arrayDelimiter, final DecodeContext context) {
+        final String nextLine = context.lines[context.currentLine];
+        final int nextDepth = DecodeHelper.getDepth(nextLine, context);
+        final String nextContent = nextLine.substring(nextDepth * context.options.indent());
+
+        if (nextDepth <= depth) {
+            // The next line is not a child of this array, the array is empty
+            validateArrayLength(header, 0, context.options.maxArraySize(), context.options.strict());
+            return Collections.emptyList();
+        }
+
+        if (LIST_ITEM_MARKER.equals(nextContent) || nextContent.startsWith(LIST_ITEM_PREFIX)) {
+            context.currentLine--;
+            return Collections.unmodifiableList(parseListArray(depth, header, context));
+        }
+
+        context.currentLine++;
+        final List<Object> result = parseArrayValues(nextContent, arrayDelimiter,
+            context.options.maxArraySize(), context.options.maxStringLength());
+        validateArrayLength(header, result.size(), context.options.maxArraySize(), context.options.strict());
+        return Collections.unmodifiableList(result);
+    }
+
     /**
      * Validates array length if declared in the header.
+     * The count check applies in strict mode only; the declared length never
+     * truncates a scope (§14.1). Resource bounds are always enforced.
      *
      * @param header       header
      * @param actualLength actual length
+     * @param maxArraySize maximum allowed array size
+     * @param strict       strict mode flag
      */
-    static void validateArrayLength(final String header, final int actualLength, final int maxArraySize) {
+    static void validateArrayLength(final String header, final int actualLength, final int maxArraySize,
+            final boolean strict) {
         final Integer declaredLength = extractLengthFromHeader(header, maxArraySize);
-        if (declaredLength != null && declaredLength != actualLength) {
+        if (strict && declaredLength != null && declaredLength != actualLength) {
             throw new IllegalArgumentException(
                 String.format("Array length mismatch: declared %d, found %d", declaredLength, actualLength));
         }
@@ -185,17 +289,6 @@ private static Integer extractLengthFromHeader(final String header, final int ma
         return null;
     }
 
-    /**
-     * Parses array values from a delimiter-separated string.
-     *
-     * @param values         the value string to parse
-     * @param arrayDelimiter array delimiter
-     * @return parsed array values
-     */
-    static List<Object> parseArrayValues(final String values, final Delimiter arrayDelimiter, final int maxArraySize) {
-        return parseArrayValues(values, arrayDelimiter, maxArraySize, Integer.MAX_VALUE);
-    }
-
     static List<Object> parseArrayValues(final String values, final Delimiter arrayDelimiter,
                                           final int maxArraySize, final int maxStringLength) {
         final List<String> rawValues = parseDelimitedValues(values, arrayDelimiter);
@@ -246,10 +339,7 @@ static List<String> parseDelimitedValues(final String input, final Delimiter arr
                 final String value = stringBuilder.toString().trim();
                 result.add(value);
                 stringBuilder.setLength(0);
-                // Skip whitespace after delimiter
-                do {
-                    i++;
-                } while (i < input.length() && Character.isWhitespace(input.charAt(i)));
+                i = skipWhitespace(input, i + 1);
             } else {
                 stringBuilder.append(currentChar);
                 i++;
@@ -264,6 +354,22 @@ static List<String> parseDelimitedValues(final String input, final Delimiter arr
         return result;
     }
 
+    /**
+     * Returns the index of the first non-whitespace character at or after
+     * the given position.
+     *
+     * @param input the input string
+     * @param start the position to scan from
+     * @return the first non-whitespace index, or the input length
+     */
+    private static int skipWhitespace(final String input, final int start) {
+        int i = start;
+        while (i < input.length() && Character.isWhitespace(input.charAt(i))) {
+            i++;
+        }
+        return i;
+    }
+
     /**
      * Parses list an array format where items are prefixed with "- ".
      * Example: items[2]:\n - item1\n - item2
@@ -277,7 +383,11 @@ private static List<Object> parseListArray(final int depth, final String header,
             final String line = context.lines[context.currentLine];
 
             if (DecodeHelper.isBlankLine(line)) {
-                if (handleBlankLineInListArray(depth, context)) {
+                // Spec §12: blank lines between the header and the first item are
+                // accepted even in strict mode
+                if (result.isEmpty()) {
+                    context.currentLine++;
+                } else if (handleBlankLineInListArray(depth, context)) {
                     shouldContinue = false;
                 }
             } else {
@@ -291,7 +401,7 @@ private static List<Object> parseListArray(final int depth, final String header,
         }
 
         if (header != null) {
-            validateArrayLength(header, result.size(), context.options.maxArraySize());
+            validateArrayLength(header, result.size(), context.options.maxArraySize(), context.options.strict());
         }
         return result;
     }