diff --git a/CHANGELOG.md b/CHANGELOG.md index e2dd7a78..b1360212 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ This project adheres to Semantic Versioning and follows a Keep a Changelog-like ## [Unreleased] +### Changed + +- Full TOON Spec 4.1 conformance: canonical number formatting, BOM stripping, comment pre-pass (§5.1), strict header validation (§5, §6, §7.3, §7.4), nested field groups in tabular arrays (§9.3), and keyed tabular form for objects of uniform objects, including the keyless root form and keyed headers on list-item hyphen lines (§9.5, §10). Conformance suite: 95/95 passing. + ## [2.0.1] - 2026-07-11 ### Added diff --git a/README.md b/README.md index d1bda161..29b6b0a3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Release](https://github.com/toon-format/toon-java/actions/workflows/release.yml/badge.svg)](https://github.com/toon-format/toon-java/actions/workflows/release.yml) [![Maven Central](https://img.shields.io/maven-central/v/dev.toonformat/jtoon.svg)](https://central.sonatype.com/artifact/dev.toonformat/jtoon) ![Coverage](.github/badges/jacoco.svg) -[![SPEC v3.3.2](https://img.shields.io/badge/spec-v3.3.2-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec) +[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec) [![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE) Compact, human-readable serialization format for LLM contexts with **30-60% token reduction** vs JSON. Combines YAML-like indentation with CSV-like tabular arrays. Working towards full compatibility with the [official TOON specification](https://github.com/toon-format/spec). diff --git a/build.gradle b/build.gradle index bd93fecd..12a68817 100644 --- a/build.gradle +++ b/build.gradle @@ -149,7 +149,7 @@ dependencies { compileOnly 'com.github.spotbugs:spotbugs-annotations:4.10.3' // NullAway + Error Prone for compile-time null safety - errorprone 'com.uber.nullaway:nullaway:0.13.7' + errorprone 'com.uber.nullaway:nullaway:0.13.8' // Pin error_prone_core to 2.42.0 (Java 17 compatible; 2.50.0+ requires Java 21) errorprone('com.google.errorprone:error_prone_core:2.42.0') { version { diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 31919739..7221ab2c 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -736,15 +736,15 @@ - - - + + + - - + + - - + + diff --git a/pmd-rules.xml b/pmd-rules.xml index c8f75451..8b52c6a7 100644 --- a/pmd-rules.xml +++ b/pmd-rules.xml @@ -61,6 +61,13 @@ + + + + + + + diff --git a/src/main/java/dev/toonformat/jtoon/decoder/ArrayDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/ArrayDecoder.java index beec689e..45b1dbe8 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/ArrayDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/ArrayDecoder.java @@ -9,6 +9,7 @@ import static dev.toonformat.jtoon.util.Constants.BACKSLASH; import static dev.toonformat.jtoon.util.Constants.COLON; import static dev.toonformat.jtoon.util.Constants.DOUBLE_QUOTE; +import static dev.toonformat.jtoon.util.Constants.LIST_ITEM_MARKER; import static dev.toonformat.jtoon.util.Constants.LIST_ITEM_PREFIX; import static dev.toonformat.jtoon.util.Headers.ARRAY_HEADER_PATTERN; import static dev.toonformat.jtoon.util.Headers.TABULAR_HEADER_PATTERN; @@ -19,11 +20,43 @@ public final class ArrayDecoder { private static final int DELIMITER_GROUP_INDEX = 3; + private static final int FIELDS_GROUP_INDEX = 4; private ArrayDecoder() { throw new UnsupportedOperationException("Utility class cannot be instantiated"); } + /** + * Spec §6: the delimiter declared inside the bracket segment of a tabular + * header must match the delimiter used by the brace field list. A header + * that declares a delimiter the field list does not use is defective. + * + * @param arrayHeader the array header starting with the bracket segment + * @return true when the header carries a mismatched delimiter declaration + */ + static boolean hasTabularDelimiterMismatch(final String arrayHeader) { + final Matcher matcher = TABULAR_HEADER_PATTERN.matcher(arrayHeader); + if (!matcher.find() || matcher.group(DELIMITER_GROUP_INDEX) == null) { + return false; + } + final char declared = matcher.group(DELIMITER_GROUP_INDEX).charAt(0); + boolean inQuotes = false; + boolean escaped = false; + for (int i = 0; i < matcher.group(FIELDS_GROUP_INDEX).length(); i++) { + final char c = matcher.group(FIELDS_GROUP_INDEX).charAt(i); + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '"') { + inQuotes = !inQuotes; + } else if (!inQuotes && c != declared && (c == ',' || c == '\t' || c == '|')) { + return true; + } + } + return false; + } + /** * Parses array from the header string and the following lines. * Detects array type (tabular, list, or primitive) and routes accordingly. @@ -85,74 +118,145 @@ static List parseArrayWithDelimiter(final String header, final int depth } if (arrayMatcher.find()) { - // In strict mode, reject bracket lengths with leading zeros (e.g. [03]) - // unless the length is exactly "0". - if (context.options.strict()) { - final String lengthStr = arrayMatcher.group(2); - if (lengthStr.length() > 1 && lengthStr.charAt(0) == '0') { - throw new IllegalArgumentException( - "Invalid array length with leading zeros: [" + lengthStr + "]"); - } - } + rejectLeadingZeroLength(arrayMatcher, context.options.strict()); final int headerEndIdx = arrayMatcher.end(); final String afterHeader = header.substring(headerEndIdx).trim(); - if (afterHeader.startsWith(COLON)) { - final String inlineContent = afterHeader.substring(1).trim(); - - if (!inlineContent.isEmpty()) { - final List result = parseArrayValues(inlineContent, arrayDelimiter, - context.options.maxArraySize(), context.options.maxStringLength()); - validateArrayLength(header, result.size(), context.options.maxArraySize()); - context.currentLine++; - return Collections.unmodifiableList(result); - } + if (hasInlineContent(afterHeader)) { + return parseInlineArray(afterHeader, header, arrayDelimiter, context); } - context.currentLine++; - if (context.currentLine < context.lines.length) { - final String nextLine = context.lines[context.currentLine]; - final int nextDepth = DecodeHelper.getDepth(nextLine, context); - final String nextContent = nextLine.substring(nextDepth * context.options.indent()); - - if (nextDepth <= depth) { - // The next line is not a child of this array, - // the array is empty - validateArrayLength(header, 0, context.options.maxArraySize()); - return Collections.emptyList(); - } + // Spec §12: blank lines between the header and the first item are + // accepted even in strict mode + skipBlankLines(context); - if (nextContent.startsWith(LIST_ITEM_PREFIX)) { - context.currentLine--; - return Collections.unmodifiableList(parseListArray(depth, header, context)); - } else { - context.currentLine++; - final List result = parseArrayValues(nextContent, arrayDelimiter, - context.options.maxArraySize(), context.options.maxStringLength()); - validateArrayLength(header, result.size(), context.options.maxArraySize()); - return Collections.unmodifiableList(result); - } + if (context.currentLine < context.lines.length) { + return parseArrayDataLine(header, depth, arrayDelimiter, context); } - final List empty = new ArrayList<>(); - validateArrayLength(header, 0, context.options.maxArraySize()); - return Collections.unmodifiableList(empty); + validateArrayLength(header, 0, context.options.maxArraySize(), context.options.strict()); + return Collections.unmodifiableList(new ArrayList<>()); + } + + // Spec §9.1/§9.2: a bare bracket pair is an empty array header + if ("[]".equals(header.trim())) { + context.currentLine++; + return Collections.emptyList(); } if (context.options.strict()) { throw new IllegalArgumentException("Invalid array header: " + header); } + context.currentLine++; return Collections.emptyList(); } + /** + * In strict mode, rejects bracket lengths with leading zeros (e.g. [03]) + * unless the length is exactly "0". + * + * @param arrayMatcher the matched array header + * @param strict strict mode flag + */ + private static void rejectLeadingZeroLength(final Matcher arrayMatcher, final boolean strict) { + if (strict) { + final String lengthStr = arrayMatcher.group(2); + if (lengthStr.length() > 1 && lengthStr.charAt(0) == '0') { + throw new IllegalArgumentException( + "Invalid array length with leading zeros: [" + lengthStr + "]"); + } + } + } + + /** + * Returns whether the header text past the bracket segment declares + * non-empty inline values ({@code header: v1,v2}). + * + * @param afterHeader the header text past the bracket segment + * @return true when inline values follow the colon + */ + private static boolean hasInlineContent(final String afterHeader) { + return afterHeader.startsWith(COLON) && !afterHeader.substring(1).isBlank(); + } + + /** + * Parses the inline values of an array header ({@code header: v1,v2}). + * + * @param afterHeader the header text past the bracket segment + * @param header the full header string + * @param arrayDelimiter array delimiter + * @param context decode context + * @return the parsed values + */ + private static List parseInlineArray(final String afterHeader, final String header, + final Delimiter arrayDelimiter, final DecodeContext context) { + final String inlineContent = afterHeader.substring(1).trim(); + final List result = parseArrayValues(inlineContent, arrayDelimiter, + context.options.maxArraySize(), context.options.maxStringLength()); + validateArrayLength(header, result.size(), context.options.maxArraySize(), context.options.strict()); + context.currentLine++; + return Collections.unmodifiableList(result); + } + + /** + * Advances the current line past blank lines following the header. + * + * @param context decode context + */ + private static void skipBlankLines(final DecodeContext context) { + do { + context.currentLine++; + } while (context.currentLine < context.lines.length + && DecodeHelper.isBlankLine(context.lines[context.currentLine])); + } + + /** + * Parses the first data line below an array header, routing list items to + * the list parser and any other content to the value splitter. + * + * @param header the full header string + * @param depth depth of the array + * @param arrayDelimiter array delimiter + * @param context decode context + * @return the parsed array values + */ + private static List parseArrayDataLine(final String header, final int depth, + final Delimiter arrayDelimiter, final DecodeContext context) { + final String nextLine = context.lines[context.currentLine]; + final int nextDepth = DecodeHelper.getDepth(nextLine, context); + final String nextContent = nextLine.substring(nextDepth * context.options.indent()); + + if (nextDepth <= depth) { + // The next line is not a child of this array, the array is empty + validateArrayLength(header, 0, context.options.maxArraySize(), context.options.strict()); + return Collections.emptyList(); + } + + if (LIST_ITEM_MARKER.equals(nextContent) || nextContent.startsWith(LIST_ITEM_PREFIX)) { + context.currentLine--; + return Collections.unmodifiableList(parseListArray(depth, header, context)); + } + + context.currentLine++; + final List result = parseArrayValues(nextContent, arrayDelimiter, + context.options.maxArraySize(), context.options.maxStringLength()); + validateArrayLength(header, result.size(), context.options.maxArraySize(), context.options.strict()); + return Collections.unmodifiableList(result); + } + /** * Validates array length if declared in the header. + * The count check applies in strict mode only; the declared length never + * truncates a scope (§14.1). Resource bounds are always enforced. * * @param header header * @param actualLength actual length + * @param maxArraySize maximum allowed array size + * @param strict strict mode flag */ - static void validateArrayLength(final String header, final int actualLength, final int maxArraySize) { + static void validateArrayLength(final String header, final int actualLength, final int maxArraySize, + final boolean strict) { final Integer declaredLength = extractLengthFromHeader(header, maxArraySize); - if (declaredLength != null && declaredLength != actualLength) { + if (strict && declaredLength != null && declaredLength != actualLength) { throw new IllegalArgumentException( String.format("Array length mismatch: declared %d, found %d", declaredLength, actualLength)); } @@ -185,17 +289,6 @@ private static Integer extractLengthFromHeader(final String header, final int ma return null; } - /** - * Parses array values from a delimiter-separated string. - * - * @param values the value string to parse - * @param arrayDelimiter array delimiter - * @return parsed array values - */ - static List parseArrayValues(final String values, final Delimiter arrayDelimiter, final int maxArraySize) { - return parseArrayValues(values, arrayDelimiter, maxArraySize, Integer.MAX_VALUE); - } - static List parseArrayValues(final String values, final Delimiter arrayDelimiter, final int maxArraySize, final int maxStringLength) { final List rawValues = parseDelimitedValues(values, arrayDelimiter); @@ -246,10 +339,7 @@ static List parseDelimitedValues(final String input, final Delimiter arr final String value = stringBuilder.toString().trim(); result.add(value); stringBuilder.setLength(0); - // Skip whitespace after delimiter - do { - i++; - } while (i < input.length() && Character.isWhitespace(input.charAt(i))); + i = skipWhitespace(input, i + 1); } else { stringBuilder.append(currentChar); i++; @@ -264,6 +354,22 @@ static List parseDelimitedValues(final String input, final Delimiter arr return result; } + /** + * Returns the index of the first non-whitespace character at or after + * the given position. + * + * @param input the input string + * @param start the position to scan from + * @return the first non-whitespace index, or the input length + */ + private static int skipWhitespace(final String input, final int start) { + int i = start; + while (i < input.length() && Character.isWhitespace(input.charAt(i))) { + i++; + } + return i; + } + /** * Parses list an array format where items are prefixed with "- ". * Example: items[2]:\n - item1\n - item2 @@ -277,7 +383,11 @@ private static List parseListArray(final int depth, final String header, final String line = context.lines[context.currentLine]; if (DecodeHelper.isBlankLine(line)) { - if (handleBlankLineInListArray(depth, context)) { + // Spec §12: blank lines between the header and the first item are + // accepted even in strict mode + if (result.isEmpty()) { + context.currentLine++; + } else if (handleBlankLineInListArray(depth, context)) { shouldContinue = false; } } else { @@ -291,7 +401,7 @@ private static List parseListArray(final int depth, final String header, } if (header != null) { - validateArrayLength(header, result.size(), context.options.maxArraySize()); + validateArrayLength(header, result.size(), context.options.maxArraySize(), context.options.strict()); } return result; } diff --git a/src/main/java/dev/toonformat/jtoon/decoder/DecodeHelper.java b/src/main/java/dev/toonformat/jtoon/decoder/DecodeHelper.java index 83f7c14b..ad2371c6 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/DecodeHelper.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/DecodeHelper.java @@ -254,4 +254,52 @@ static void validateNoMultiplePrimitivesAtRoot(final DecodeContext context) { } } + /** + * Ensures no unconsumed lines remain after the root form was parsed. + * The root form spans the whole document (§5); trailing content must not be + * silently discarded. In strict mode any leftover line is an error. In + * non-strict mode a scalar line outside root primitive position is still an + * error in both modes alike (§5.2), while leftover key-value lines are ignored. + * + * @param context decode an object to deal with lines, delimiter and options + */ + static void validateNoTrailingContent(final DecodeContext context) { + while (context.currentLine < context.lines.length) { + final String line = context.lines[context.currentLine]; + if (isBlankLine(line)) { + context.currentLine++; + continue; + } + if (context.options.strict()) { + throw new IllegalArgumentException( + "Unexpected content after root form at line " + (context.currentLine + 1)); + } + final int depth = getDepth(line, context); + final String content = line.substring(depth * context.options.indent()); + if (findUnquotedColon(content) < 0) { + // Spec §5.2: a scalar line outside root primitive position is + // an error in strict and non-strict mode alike. + throw new FatalDecodeException( + "Bare token line outside root primitive position at line " + (context.currentLine + 1)); + } + context.currentLine++; + } + } + + /** + * Skips or rejects an over-indented line that jumps past the expected + * depth (§14.2). + * + * @param context decode an object to deal with lines, delimiter, and options + * @param lineDepth the depth of the over-indented line + * @throws IllegalArgumentException in strict mode + */ + static void processOverIndentedLine(final DecodeContext context, final int lineDepth) { + if (context.options.strict()) { + throw new IllegalArgumentException( + "Over-indented line at " + (context.currentLine + 1) + " (depth " + lineDepth + ")"); + } + context.currentLine++; + } + } diff --git a/src/main/java/dev/toonformat/jtoon/decoder/FatalDecodeException.java b/src/main/java/dev/toonformat/jtoon/decoder/FatalDecodeException.java new file mode 100644 index 00000000..d8bcf4a6 --- /dev/null +++ b/src/main/java/dev/toonformat/jtoon/decoder/FatalDecodeException.java @@ -0,0 +1,16 @@ +package dev.toonformat.jtoon.decoder; + +/** + * Signals a decode defect that is fatal in strict and non-strict mode alike: + * a bare scalar line outside root primitive position (§5.2) and characters + * after a closing quote (§7.4). {@link ValueDecoder#decode} rethrows it even + * in lenient mode instead of converting it to {@code null}. + */ +final class FatalDecodeException extends IllegalArgumentException { + + private static final long serialVersionUID = 1L; + + FatalDecodeException(final String message) { + super(message); + } +} diff --git a/src/main/java/dev/toonformat/jtoon/decoder/KeyDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/KeyDecoder.java index b8eb6f52..07465048 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/KeyDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/KeyDecoder.java @@ -2,14 +2,12 @@ import dev.toonformat.jtoon.Delimiter; import dev.toonformat.jtoon.PathExpansion; +import dev.toonformat.jtoon.util.Headers; import dev.toonformat.jtoon.util.StringEscaper; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.MatchResult; -import static dev.toonformat.jtoon.util.Headers.KEYED_ARRAY_PATTERN; /** * Handles decoding of key values/arrays to JSON format. @@ -27,18 +25,35 @@ private KeyDecoder() { * * @param result result * @param content the content string to parse - * @param originalKey the original Key + * @param keyedHeader the keyed header match for the content * @param parentDepth parent depth of keyed array line * @param context decode an object to deal with lines, delimiter and options */ - static void processKeyedArrayLine(final Map result, final String content, final String originalKey, + static void processKeyedArrayLine(final Map result, final String content, + final Headers.KeyedHeaderMatch keyedHeader, final int parentDepth, final DecodeContext context) { - final String key = StringEscaper.unescape(originalKey); - final String arrayHeader = content.substring(originalKey.length()); + if (keyedHeader.keyed()) { + final Object keyedValue = KeyedObjectDecoder.parseKeyedTabularObject( + content, keyedHeader, parentDepth + 2, context); + putKeyedValueIntoMap(result, keyedHeader, keyedValue, context); + return; + } + + final String key = StringEscaper.unescape(keyedHeader.key()); + final String arrayHeader = content.substring(keyedHeader.keyEnd()); + + // Spec §6: a keyed tabular header whose bracket and brace segments + // declare different delimiters is defective; in non-strict mode the + // whole line falls through and decodes as an ordinary key-value pair. + if (!context.options.strict() && ArrayDecoder.hasTabularDelimiterMismatch(arrayHeader)) { + processKeyValueLine(result, content, parentDepth + 1, context); + return; + } + final List arrayValue = ArrayDecoder.parseArray(arrayHeader, parentDepth + 1, context); // Handle path expansion for array keys - if (shouldExpandKey(originalKey, context)) { + if (shouldExpandKey(keyedHeader.key(), context)) { expandPathIntoMap(result, key, arrayValue, context); } else { // Check for conflicts with existing expanded paths @@ -48,6 +63,28 @@ static void processKeyedArrayLine(final Map result, final String } } + /** + * Puts a keyed value into a map under the match's key, honoring path + * expansion, path-expansion conflicts and duplicate keys. + * + * @param map the map to put the keyed value into + * @param keyedHeader the keyed header match carrying the key + * @param value the value to put + * @param context decode an object to deal with lines, delimiter and options + */ + static void putKeyedValueIntoMap(final Map map, + final Headers.KeyedHeaderMatch keyedHeader, final Object value, final DecodeContext context) { + final String originalKey = keyedHeader.key().trim(); + final String key = StringEscaper.unescape(originalKey); + if (shouldExpandKey(originalKey, context)) { + expandPathIntoMap(map, key, value, context); + } else { + DecodeHelper.checkPathExpansionConflict(map, key, value, context); + DecodeHelper.checkDuplicateKey(map, key, context); + map.put(key, value); + } + } + /** * Expands a dotted key into a nested object structure. * @@ -188,39 +225,17 @@ static boolean shouldExpandKey(final String key, final DecodeContext context) { * @return the parsed value (Map, List, or primitive) */ private static Object parseKeyValue(final String value, final int depth, final DecodeContext context) { - // Check if the next line is nested (deeper indentation) - if (context.currentLine + 1 < context.lines.length) { - final int nextDepth = DecodeHelper.getDepth(context.lines[context.currentLine + 1], context); - if (nextDepth > depth) { - context.currentLine++; - // parseNestedObject manages the currentLine, so we don't increment here - return ObjectDecoder.parseNestedObject(depth, context); - } else { - // If the value is empty, create an empty object; otherwise parse as primitive - final Object parsedValue; - if (value.isBlank()) { - parsedValue = new LinkedHashMap<>(); - } else if ("[]".equals(value)) { - parsedValue = List.of(); - } else { - parsedValue = PrimitiveDecoder.parse(value, context); - } - context.currentLine++; - return parsedValue; - } - } else { - // If the value is empty, create an empty object; otherwise parse as primitive - final Object parsedValue; - if (value.isBlank()) { - parsedValue = new LinkedHashMap<>(); - } else if ("[]".equals(value)) { - parsedValue = List.of(); - } else { - parsedValue = PrimitiveDecoder.parse(value, context); - } - context.currentLine++; - return parsedValue; + return ObjectDecoder.parseValueWithNestedScope(value, depth, context, KeyDecoder::parseScalarValue); + } + + private static Object parseScalarValue(final String value, final DecodeContext context) { + if (value.isBlank()) { + return new LinkedHashMap<>(); + } + if ("[]".equals(value)) { + return List.of(); } + return PrimitiveDecoder.parse(value, context); } /** @@ -268,18 +283,41 @@ static Object parseKeyValuePair(final String key, final String value, final int /** * Parses a keyed array value (e.g., "items[2]{id,name}:"). * - * @param keyedArray keyed array - * @param content the content string to parse - * @param depth the depth of the keyed array value - * @param context decode an object to deal with lines, delimiter, and options + * @param keyedHeader keyed header match + * @param content the content string to parse + * @param depth the depth of the keyed array value + * @param context decode an object to deal with lines, delimiter and options * @return parsed keyed array value */ - static Object parseKeyedArrayValue(final MatchResult keyedArray, final String content, + static Object parseKeyedArrayValue(final Headers.KeyedHeaderMatch keyedHeader, final String content, final int depth, final DecodeContext context) { - final String group1 = keyedArray.group(1); - final String originalKey = group1.trim(); + if (keyedHeader.keyed()) { + final Object keyedValue = KeyedObjectDecoder.parseKeyedTabularObject(content, keyedHeader, depth + 1, + context); + final Map obj = new LinkedHashMap<>(); + putKeyedValueIntoMap(obj, keyedHeader, keyedValue, context); + + // Continue parsing root-level fields if at depth 0 + if (depth == 0) { + ObjectDecoder.parseRootObjectFields(obj, depth, context); + } + return obj; + } + + final String originalKey = keyedHeader.key().trim(); final String key = StringEscaper.unescape(originalKey); - final String arrayHeader = content.substring(group1.length()); + final String arrayHeader = content.substring(keyedHeader.keyEnd()); + + // Spec §6: a keyed tabular header whose bracket and brace segments + // declare different delimiters is defective; in non-strict mode the + // whole line falls through and decodes as an ordinary key-value pair. + if (!context.options.strict() && ArrayDecoder.hasTabularDelimiterMismatch(arrayHeader)) { + final int colonIdx = DecodeHelper.findUnquotedColon(content); + if (colonIdx > 0) { + return parseKeyValuePair(content.substring(0, colonIdx).trim(), + content.substring(colonIdx + 1).trim(), depth, depth == 0, context); + } + } final List arrayValue = ArrayDecoder.parseArray(arrayHeader, depth, context); final Map obj = new LinkedHashMap<>(); @@ -312,15 +350,29 @@ static Object parseKeyedArrayValue(final MatchResult keyedArray, final String co */ static boolean parseKeyedArrayField(final String fieldContent, final Map item, final int depth, final DecodeContext context) { - final Matcher keyedArray = KEYED_ARRAY_PATTERN.matcher(fieldContent); - if (!keyedArray.matches()) { + final Headers.KeyedHeaderMatch keyedHeader = Headers.matchKeyedArrayHeader(fieldContent); + if (keyedHeader == null) { + return false; + } + + if (keyedHeader.keyed()) { + final Object keyedValue = KeyedObjectDecoder.parseKeyedTabularObject( + fieldContent, keyedHeader, depth + 3, context); + putKeyedValueIntoMap(item, keyedHeader, keyedValue, context); + return true; + } + + // Spec §6: a keyed tabular header whose bracket and brace segments + // declare different delimiters is defective; in non-strict mode the + // field falls through to ordinary key-value parsing. + if (!context.options.strict() + && ArrayDecoder.hasTabularDelimiterMismatch(fieldContent.substring(keyedHeader.keyEnd()))) { return false; } - final String group1 = keyedArray.group(1); - final String originalKey = group1.trim(); + final String originalKey = keyedHeader.key().trim(); final String key = StringEscaper.unescape(originalKey); - final String arrayHeader = fieldContent.substring(group1.length()); + final String arrayHeader = fieldContent.substring(keyedHeader.keyEnd()); // For nested arrays in list items, default to comma delimiter if not specified final Delimiter nestedArrayDelimiter = ArrayDecoder.extractDelimiterFromHeader(arrayHeader, context); diff --git a/src/main/java/dev/toonformat/jtoon/decoder/KeyedObjectDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/KeyedObjectDecoder.java new file mode 100644 index 00000000..9f4df922 --- /dev/null +++ b/src/main/java/dev/toonformat/jtoon/decoder/KeyedObjectDecoder.java @@ -0,0 +1,214 @@ +package dev.toonformat.jtoon.decoder; + +import org.jspecify.annotations.Nullable; +import dev.toonformat.jtoon.Delimiter; +import dev.toonformat.jtoon.util.Headers; +import dev.toonformat.jtoon.util.StringEscaper; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Handles decoding of keyed tabular objects (§9.5) to JSON format. + * + *

A keyed tabular object is a keyed array header with a keyed marker + * ({@code key[N:]{field…}:} or, at document root only, {@code [N:]{field…}:}) + * followed by entry rows at header depth + 1, each starting with an entry key + * and a colon before its cell values.

+ */ +public final class KeyedObjectDecoder { + + private KeyedObjectDecoder() { + throw new UnsupportedOperationException("Utility class cannot be instantiated"); + } + + /** + * Parses a keyed tabular object, e.g. {@code servers[2:]{host,port}:} + * followed by entry rows {@code alpha: a.example.com,8080}. + * + * @param content the header line content + * @param header the keyed header match for the line + * @param entryDepth the depth of the entry rows (header depth + 1; for + * hyphen-line items one level below the header, §10) + * @param context decode an object to deal with lines, delimiter and options + * @return the parsed keyed object, an entry-keyed map of row maps + */ + static Map parseKeyedTabularObject(final String content, + final Headers.KeyedHeaderMatch header, final int entryDepth, final DecodeContext context) { + validateKeyedHeader(content, header, context); + + final String fieldsSpec = content.substring(header.fieldsStart() + 1, header.headerEnd() - 1); + final Delimiter arrayDelimiter = delimiterFromChar(header.delimiter(), context); + + final List fields = + TabularArrayDecoder.parseTabularKeys(fieldsSpec, arrayDelimiter, context); + + // Spec §9.3: a duplicate field name within one field list is a header + // defect, diagnosed from the header line alone. + if (context.options.strict()) { + TabularArrayDecoder.validateNoDuplicateFields(fields, context); + } + + final Map result = new LinkedHashMap<>(); + context.currentLine++; + + while (context.currentLine < context.lines.length) { + final LineHandling handling = handleNextLine(result, entryDepth, context); + if (handling == LineHandling.STOP) { + break; + } + if (handling == LineHandling.PROCESS) { + processEntryLine(context.lines[context.currentLine], entryDepth, + fields, arrayDelimiter, result, context); + context.currentLine++; + } + } + + // Spec §9.5: the declared entry count must match in strict mode + if (context.options.strict() && result.size() != header.declaredLength()) { + throw new IllegalArgumentException( + String.format("Keyed object entry count (%d) does not match declared length (%d)", + result.size(), header.declaredLength())); + } + return result; + } + + /** + * Validates the keyed header shape: a field list must be present and no + * inline content may follow the header. + * + * @param content the header line content + * @param header the keyed header match for the line + * @param context decode an object to deal with lines, delimiter and options + * @throws IllegalArgumentException for a defective header + */ + private static void validateKeyedHeader(final String content, + final Headers.KeyedHeaderMatch header, final DecodeContext context) { + if (header.fieldsStart() < 0) { + throw new IllegalArgumentException( + "Keyed header requires a field list at line " + (context.currentLine + 1)); + } + if (!content.substring(header.headerEnd() + 1).isBlank()) { + throw new IllegalArgumentException( + "Inline content after keyed header at line " + (context.currentLine + 1)); + } + } + + /** + * Returns whether parsing stops at a blank line: at the end of the input + * or when the next non-blank line sits outside the keyed object. Blank + * lines inside the object are rejected in strict mode (§12). + * + * @param result the rows parsed so far + * @param entryDepth the depth of the entry rows + * @param context decode an object to deal with lines, delimiter and options + * @return true when the blank line terminates the object + */ + private static boolean shouldStopAtBlankLine(final Map result, final int entryDepth, + final DecodeContext context) { + final int nextNonBlank = DecodeHelper.findNextNonBlankLine(context.currentLine + 1, context); + if (nextNonBlank >= context.lines.length) { + return true; // EOF - terminate + } + final int nextDepth = DecodeHelper.getDepth(context.lines[nextNonBlank], context); + if (nextDepth <= entryDepth - 1) { + return true; // outside the object - terminate + } + // Spec §12: blank lines between the header and the first entry + // row are accepted; later blank lines inside the object are a + // defect in strict mode. + if (!result.isEmpty() && context.options.strict()) { + throw new IllegalArgumentException( + "Blank line inside keyed object at line " + (context.currentLine + 1)); + } + return false; + } + + /** + * How the next line of a keyed object is handled by the parse loop. + */ + private enum LineHandling { STOP, SKIP, PROCESS } + + /** + * Classifies the current line of a keyed tabular object: terminates the + * object at blank-line/EOF boundaries or shallower lines, skips blank and + * over-indented lines (§14.2), and passes entry rows through. + * + * @param result the rows parsed so far + * @param entryDepth the depth of the entry rows + * @param context decode an object to deal with lines, delimiter and options + * @return the handling to apply to the current line + */ + private static LineHandling handleNextLine(final Map result, final int entryDepth, + final DecodeContext context) { + final String line = context.lines[context.currentLine]; + + if (DecodeHelper.isBlankLine(line)) { + if (shouldStopAtBlankLine(result, entryDepth, context)) { + return LineHandling.STOP; + } + context.currentLine++; + return LineHandling.SKIP; + } + + final int lineDepth = DecodeHelper.getDepth(line, context); + if (lineDepth < entryDepth) { + return LineHandling.STOP; + } + if (lineDepth > entryDepth) { + DecodeHelper.processOverIndentedLine(context, lineDepth); + return LineHandling.SKIP; + } + return LineHandling.PROCESS; + } + + /** + * Parses one entry row of a keyed tabular object, splitting the row at + * its first unquoted colon (§9.5). A row without a colon is rejected in + * strict mode; otherwise the caller skips it. + * + * @param line the entry row line + * @param entryDepth the depth of the entry rows + * @param fields the declared field nodes + * @param arrayDelimiter the active delimiter + * @param result the entry-keyed result map + * @param context decode an object to deal with lines, delimiter and options + */ + private static void processEntryLine(final String line, final int entryDepth, + final List fields, final Delimiter arrayDelimiter, + final Map result, final DecodeContext context) { + final String entryContent = line.substring(entryDepth * context.options.indent()); + // Spec §9.5: an entry row splits at its first unquoted colon; the + // remainder is parsed as a tabular row with the active delimiter. + final int colonIdx = DecodeHelper.findUnquotedColon(entryContent); + if (colonIdx <= 0) { + if (context.options.strict()) { + throw new IllegalArgumentException( + "Missing colon in keyed entry at line " + (context.currentLine + 1)); + } + return; + } + + final String entryKey = StringEscaper.unescape(entryContent.substring(0, colonIdx).trim()); + final Map entry = TabularArrayDecoder.parseTabularRow( + entryContent.substring(colonIdx + 1), fields, arrayDelimiter, context); + + DecodeHelper.checkDuplicateKey(result, entryKey, context); + result.put(entryKey, entry); + } + + private static Delimiter delimiterFromChar(@Nullable final Character delimiter, + final DecodeContext context) { + if (delimiter == null) { + return context.delimiter; + } + final char c = delimiter; + if (c == Delimiter.TAB.getValue()) { + return Delimiter.TAB; + } + if (c == Delimiter.PIPE.getValue()) { + return Delimiter.PIPE; + } + return Delimiter.COMMA; + } +} diff --git a/src/main/java/dev/toonformat/jtoon/decoder/ListItemDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/ListItemDecoder.java index ff06e616..bc47490e 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/ListItemDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/ListItemDecoder.java @@ -1,21 +1,26 @@ package dev.toonformat.jtoon.decoder; import dev.toonformat.jtoon.Delimiter; +import dev.toonformat.jtoon.util.Headers; import dev.toonformat.jtoon.util.StringEscaper; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; +import java.util.regex.Pattern; import static dev.toonformat.jtoon.util.Constants.LIST_ITEM_MARKER; import static dev.toonformat.jtoon.util.Constants.OPEN_BRACKET; -import static dev.toonformat.jtoon.util.Headers.KEYED_ARRAY_PATTERN; /** * Handles decoding of TOON list item to JSON format. */ public final class ListItemDecoder { + // Spec §6: a keyless array header is valid as a list item only in its + // plain form ([N]: or []); a fields-bearing ([N]{...}:) or keyed + // ([N:]{...}:) keyless header is a defect. + private static final Pattern KEYLESS_FIELDS_HEADER = Pattern.compile("^\\[[^]]*]\\s*\\{"); + private ListItemDecoder() { throw new UnsupportedOperationException("Utility class cannot be instantiated"); } @@ -54,13 +59,7 @@ public static void processListArrayItem(final String line, final int lineDepth, * @return parsed item (scalar value or object) */ static Object parseListItem(final String content, final int depth, final DecodeContext context) { - // Handle empty item: just "-" or "- " - final String itemContent; - if (content.length() > 2) { - itemContent = content.substring(2).trim(); - } else { - itemContent = ""; - } + final String itemContent = extractItemContent(content); // Handle empty item: just "-" if (itemContent.isEmpty()) { @@ -70,42 +69,16 @@ static Object parseListItem(final String content, final int depth, final DecodeC // Check for standalone array (e.g., "[2]: 1,2") if (itemContent.startsWith(OPEN_BRACKET)) { - // For nested arrays in list items, default to comma delimiter if not specified - final Delimiter nestedArrayDelimiter = ArrayDecoder.extractDelimiterFromHeader(itemContent, context); - // parseArrayWithDelimiter handles currentLine increment internally - // For inline arrays, it increments. For multi-line arrays, parseListArray - // handles it. - // We need to increment here only if it was an inline array that we just parsed - // Actually, parseArrayWithDelimiter always handles currentLine, so we don't - // need to increment - return ArrayDecoder.parseArrayWithDelimiter(itemContent, depth + 1, nestedArrayDelimiter, context); + return parseStandaloneArrayItem(itemContent, depth, context); } // Check for keyed array pattern (e.g., "tags[3]: a,b,c" or "data[2]{id}: ...") - final Matcher keyedArray = KEYED_ARRAY_PATTERN.matcher(itemContent); - if (keyedArray.matches()) { - final String originalKey = keyedArray.group(1).trim(); - final String key = StringEscaper.unescape(originalKey); - final String arrayHeader = itemContent.substring(keyedArray.group(1).length()); - - // For nested arrays in list items, default to comma delimiter if not specified - final Delimiter nestedArrayDelimiter = ArrayDecoder.extractDelimiterFromHeader(arrayHeader, context); - final List arrayValue = ArrayDecoder.parseArrayWithDelimiter( - arrayHeader, depth + 2, nestedArrayDelimiter, context - ); - - final Map item = new LinkedHashMap<>(); - DecodeHelper.checkDuplicateKey(item, key, context); - item.put(key, arrayValue); - - // parseArrayWithDelimiter manages currentLine correctly: - // - For inline arrays, it increments currentLine - // - For multi-line arrays (list/tabular), the array parsers leave currentLine - // at the line after the array - // So we don't need to increment here. Just parse additional fields. - parseListItemFields(item, depth, context); - - return item; + final Headers.KeyedHeaderMatch keyedHeader = Headers.matchKeyedArrayHeader(itemContent); + if (keyedHeader != null && keyedHeader.keyed()) { + return parseKeyedTabularListItem(itemContent, keyedHeader, depth, context); + } + if (keyedHeader != null && !isKeyedMismatchFallThrough(itemContent, keyedHeader, context)) { + return parseKeyedArrayListItem(itemContent, keyedHeader, depth, context); } final int colonIdx = DecodeHelper.findUnquotedColon(itemContent); @@ -116,6 +89,134 @@ static Object parseListItem(final String content, final int depth, final DecodeC return PrimitiveDecoder.parse(itemContent, context); } + return parseObjectListItem(itemContent, colonIdx, depth, context); + } + + /** + * Extracts the item content past the leading "- " marker. + * + * @param content the list item line content + * @return the trimmed item content, or an empty string + */ + private static String extractItemContent(final String content) { + if (content.length() > 2) { + return content.substring(2).trim(); + } + return ""; + } + + /** + * Parses a standalone array item, validating the keyless header form. + * + * @param itemContent the item content starting with the bracket segment + * @param depth the depth of the list item + * @param context decode an object to deal with lines, delimiter and options + * @return the parsed array + */ + private static Object parseStandaloneArrayItem(final String itemContent, final int depth, + final DecodeContext context) { + // Keyless headers are valid as list items only without a field + // list; [2]{x}: and [2:]{v}: are defects (§5, §6) + if (context.options.strict() && KEYLESS_FIELDS_HEADER.matcher(itemContent).find()) { + throw new IllegalArgumentException( + "Keyless array header with field list only valid at document root at line " + + (context.currentLine + 1)); + } + final Delimiter nestedArrayDelimiter = ArrayDecoder.extractDelimiterFromHeader(itemContent, context); + return ArrayDecoder.parseArrayWithDelimiter(itemContent, depth + 1, nestedArrayDelimiter, context); + } + + /** + * Returns whether a keyed header with a mismatched tabular delimiter + * falls through to ordinary key-value parsing in non-strict mode (§6). + * + * @param itemContent the item content + * @param keyedHeader the matched keyed header + * @param context decode an object to deal with lines, delimiter and options + * @return true when the line falls through to key-value parsing + */ + private static boolean isKeyedMismatchFallThrough(final String itemContent, + final Headers.KeyedHeaderMatch keyedHeader, final DecodeContext context) { + return !keyedHeader.keyed() + && !context.options.strict() + && ArrayDecoder.hasTabularDelimiterMismatch(itemContent.substring(keyedHeader.keyEnd())); + } + + /** + * Parses a keyed tabular list item, keeping its entry rows at depth + 3. + * + * @param itemContent the item content + * @param keyedHeader the matched keyed header + * @param depth the depth of the list item + * @param context decode an object to deal with lines, delimiter and options + * @return the parsed item map + */ + private static Map parseKeyedTabularListItem(final String itemContent, + final Headers.KeyedHeaderMatch keyedHeader, final int depth, final DecodeContext context) { + final String originalKey = keyedHeader.key().trim(); + final String key = StringEscaper.unescape(originalKey); + final Map item = new LinkedHashMap<>(); + + // Spec §9.5/§10: a keyed tabular object on the hyphen line keeps + // its entry rows at document depth + 3 (header on depth + 1) and + // its sibling fields at depth + 2. + final Object keyedValue = KeyedObjectDecoder.parseKeyedTabularObject( + itemContent, keyedHeader, depth + 3, context); + DecodeHelper.checkDuplicateKey(item, key, context); + item.put(key, keyedValue); + + // parseKeyedTabularObject manages currentLine: entry rows and the + // sibling lines that follow them are left to parseListItemFields. + parseListItemFields(item, depth, context); + + return item; + } + + /** + * Parses a keyed array list item ({@code - tags[2]: a,b}). + * + * @param itemContent the item content + * @param keyedHeader the matched keyed header + * @param depth the depth of the list item + * @param context decode an object to deal with lines, delimiter and options + * @return the parsed item map + */ + private static Map parseKeyedArrayListItem(final String itemContent, + final Headers.KeyedHeaderMatch keyedHeader, final int depth, final DecodeContext context) { + final String originalKey = keyedHeader.key().trim(); + final String key = StringEscaper.unescape(originalKey); + final String arrayHeader = itemContent.substring(keyedHeader.keyEnd()); + + final Delimiter nestedArrayDelimiter = ArrayDecoder.extractDelimiterFromHeader(arrayHeader, context); + final List arrayValue = ArrayDecoder.parseArrayWithDelimiter( + arrayHeader, depth + 2, nestedArrayDelimiter, context + ); + + final Map item = new LinkedHashMap<>(); + DecodeHelper.checkDuplicateKey(item, key, context); + item.put(key, arrayValue); + + // parseArrayWithDelimiter manages currentLine correctly: + // - For inline arrays, it increments currentLine + // - For multi-line arrays (list/tabular), the array parsers leave currentLine + // at the line after the array + // So we don't need to increment here. Just parse additional fields. + parseListItemFields(item, depth, context); + + return item; + } + + /** + * Parses an object list item ({@code - key: value}). + * + * @param itemContent the item content + * @param colonIdx the index of the key-value colon + * @param depth the depth of the list item + * @param context decode an object to deal with lines, delimiter and options + * @return the parsed item map + */ + private static Map parseObjectListItem(final String itemContent, final int colonIdx, + final int depth, final DecodeContext context) { // Object item: - key: value final String key = StringEscaper.unescape(itemContent.substring(0, colonIdx).trim()); final String value = itemContent.substring(colonIdx + 1).trim(); @@ -156,22 +257,36 @@ private static void parseListItemFields(final Map item, } if (lineDepth == depth + 2) { - final String fieldContent = line.substring((depth + 2) * context.options.indent()); - - // Try to parse as a keyed array first, then as a key-value pair - boolean wasParsed = KeyDecoder.parseKeyedArrayField(fieldContent, item, depth, context); - if (!wasParsed) { - wasParsed = KeyDecoder.parseKeyValueField(fieldContent, item, depth, context); - } - - // If neither pattern matched, skip this line to avoid an infinite loop - if (!wasParsed) { - context.currentLine++; - } + processListItemFieldLine(item, line, depth, context); } else { - // lineDepth > depth + 2, skip this line - context.currentLine++; + // lineDepth > depth + 2: over-indented line (§14.2) + DecodeHelper.processOverIndentedLine(context, lineDepth); } } } + + /** + * Processes a sibling field line of a list item, falling back to + * key-value parsing when the keyed array pattern does not match. + * + * @param item the item to fill + * @param line the field line + * @param depth the depth of the item + * @param context decode an object to deal with lines, delimiter and options + */ + private static void processListItemFieldLine(final Map item, final String line, + final int depth, final DecodeContext context) { + final String fieldContent = line.substring((depth + 2) * context.options.indent()); + + // Try to parse as a keyed array first, then as a key-value pair + boolean wasParsed = KeyDecoder.parseKeyedArrayField(fieldContent, item, depth, context); + if (!wasParsed) { + wasParsed = KeyDecoder.parseKeyValueField(fieldContent, item, depth, context); + } + + // If neither pattern matched, skip this line to avoid an infinite loop + if (!wasParsed) { + context.currentLine++; + } + } } diff --git a/src/main/java/dev/toonformat/jtoon/decoder/ObjectDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/ObjectDecoder.java index 75f97ca4..8fcbdb90 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/ObjectDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/ObjectDecoder.java @@ -1,11 +1,12 @@ package dev.toonformat.jtoon.decoder; +import dev.toonformat.jtoon.util.Headers; import dev.toonformat.jtoon.util.StringEscaper; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import static dev.toonformat.jtoon.util.Headers.KEYED_ARRAY_PATTERN; +import java.util.function.BiFunction; +import static dev.toonformat.jtoon.util.Constants.OPEN_BRACKET; /** * Handles decoding of TOON objects to JSON format. @@ -52,6 +53,13 @@ private static Map doParseNestedObject(final int parentDepth, fi if (depth == parentDepth + 1) { processDirectChildLine(result, line, parentDepth, depth, context); + } else if (depth > parentDepth + 1) { + // Over-indented line jumps past the expected depth (§14.2) + if (context.options.strict()) { + throw new IllegalArgumentException( + "Over-indented line at " + (context.currentLine + 1) + " (depth " + depth + ")"); + } + context.currentLine++; } else { context.currentLine++; } @@ -68,10 +76,18 @@ private static Map doParseNestedObject(final int parentDepth, fi private static void processDirectChildLine(final Map result, final String line, final int parentDepth, final int depth, final DecodeContext context) { final String content = line.substring((parentDepth + 1) * context.options.indent()); - final Matcher keyedArray = KEYED_ARRAY_PATTERN.matcher(content); - if (keyedArray.find()) { - KeyDecoder.processKeyedArrayLine(result, content, keyedArray.group(1), parentDepth, context); + // Spec §5/§6: keyless array headers are valid only as the document's + // root header or as list items; an object field position is a defect. + if (content.startsWith(OPEN_BRACKET) && context.options.strict()) { + throw new IllegalArgumentException( + "Keyless array header only valid at document root at line " + (context.currentLine + 1)); + } + + final Headers.KeyedHeaderMatch keyedHeader = Headers.matchKeyedArrayHeader(content); + + if (keyedHeader != null) { + KeyDecoder.processKeyedArrayLine(result, content, keyedHeader, parentDepth, context); } else { KeyDecoder.processKeyValueLine(result, content, depth, context); } @@ -85,13 +101,8 @@ private static void processDirectChildLine(final Map result, fin * @param context decode an object to deal with lines, delimiter and options */ static void parseRootObjectFields(final Map obj, final int depth, final DecodeContext context) { - while (context.currentLine < context.lines.length) { + while (isRootFieldLine(depth, context)) { final String line = context.lines[context.currentLine]; - final int lineDepth = DecodeHelper.getDepth(line, context); - - if (lineDepth != depth) { - return; - } // Skip blank lines if (DecodeHelper.isBlankLine(line)) { @@ -101,39 +112,99 @@ static void parseRootObjectFields(final Map obj, final int depth final String content = line.substring(depth * context.options.indent()); - final Matcher keyedArray = KEYED_ARRAY_PATTERN.matcher(content); - if (keyedArray.matches()) { - processRootKeyedArrayLine(obj, content, keyedArray.group(1), depth, context); - } else { - final int colonIdx = DecodeHelper.findUnquotedColon(content); - if (colonIdx > 0) { - final String key = content.substring(0, colonIdx).trim(); - final String value = content.substring(colonIdx + 1).trim(); - - KeyDecoder.parseKeyValuePairIntoMap(obj, key, value, depth, context); - } else { - return; - } + if (!processRootFieldLine(obj, content, depth, context)) { + return; } } } + /** + * Returns whether the current line is a root field line at the given + * depth, staying within the line buffer. + * + * @param depth the expected root field depth + * @param context decode an object to deal with lines, delimiter and options + * @return true when the current line sits at the root field depth + */ + private static boolean isRootFieldLine(final int depth, final DecodeContext context) { + return context.currentLine < context.lines.length + && DecodeHelper.getDepth(context.lines[context.currentLine], context) == depth; + } + + /** + * Processes a single root field line. Returns false when the line is not + * a root field, terminating the root field scan. + * + * @param obj the string key-value pairs + * @param content the content string to parse + * @param depth the depth of the object field + * @param context decode an object to deal with lines, delimiter and options + * @return false when the line ends the root field scan + */ + private static boolean processRootFieldLine(final Map obj, final String content, + final int depth, final DecodeContext context) { + // Spec §5/§6: a keyless header is only valid as the document's + // root header, i.e. the first line; at any later depth-0 position + // it is a defect. + if (content.startsWith(OPEN_BRACKET) && context.options.strict()) { + throw new IllegalArgumentException( + "Keyless array header only valid as root header at line " + (context.currentLine + 1)); + } + + final Headers.KeyedHeaderMatch keyedHeader = Headers.matchKeyedArrayHeader(content); + if (keyedHeader != null) { + processRootKeyedArrayLine(obj, content, keyedHeader, depth, context); + return true; + } + + final int colonIdx = DecodeHelper.findUnquotedColon(content); + if (colonIdx > 0) { + final String key = content.substring(0, colonIdx).trim(); + final String value = content.substring(colonIdx + 1).trim(); + + KeyDecoder.parseKeyValuePairIntoMap(obj, key, value, depth, context); + return true; + } + return false; + } + /** * Processes a keyed array line in root object fields. * * @param objectMap the string key-value pairs * @param content the content string to parse - * @param originalKey the original Key + * @param keyedHeader the keyed header match for the content * @param depth the depth of the object field * @param context decode an object to deal with lines, delimiter and options */ private static void processRootKeyedArrayLine(final Map objectMap, - final String content, final String originalKey, final int depth, + final String content, final Headers.KeyedHeaderMatch keyedHeader, final int depth, final DecodeContext context) { + if (keyedHeader.keyed()) { + final Object keyedValue = KeyedObjectDecoder.parseKeyedTabularObject(content, keyedHeader, depth + 1, + context); + KeyDecoder.putKeyedValueIntoMap(objectMap, keyedHeader, keyedValue, context); + return; + } + + final String originalKey = keyedHeader.key(); final String originalKeyTrimmed = originalKey.trim(); final String key = StringEscaper.unescape(originalKey); final String arrayHeader = content.substring(originalKey.length()); + // Spec §6: a keyed tabular header whose bracket and brace segments + // declare different delimiters is defective; in non-strict mode the + // whole line falls through and decodes as an ordinary key-value pair. + if (!context.options.strict() && ArrayDecoder.hasTabularDelimiterMismatch(arrayHeader)) { + final int colonIdx = DecodeHelper.findUnquotedColon(content); + if (colonIdx > 0) { + KeyDecoder.parseKeyValuePairIntoMap(objectMap, + content.substring(0, colonIdx).trim(), + content.substring(colonIdx + 1).trim(), depth, context); + return; + } + } + final List arrayValue = ArrayDecoder.parseArray(arrayHeader, depth, context); // Handle path expansion for array keys @@ -176,33 +247,80 @@ static Object parseBareScalarValue(final String content, final int depth, final * @return the parsed value (Map, List, or primitive) */ static Object parseFieldValue(final String fieldValue, final int fieldDepth, final DecodeContext context) { - // Check if the next line is nested + return parseValueWithNestedScope(fieldValue, fieldDepth, context, ObjectDecoder::parseFieldScalar); + } + + /** + * Parses a field value that does not open a nested scope: a blank value + * becomes an empty object, any other value a primitive. + * + * @param value the value string to parse + * @param context decode an object to deal with lines, delimiter and options + * @return the parsed value (Map or primitive) + */ + private static Object parseFieldScalar(final String value, final DecodeContext context) { + if (value.isBlank()) { + return new LinkedHashMap<>(); + } + return PrimitiveDecoder.parse(value, context); + } + + /** + * Parses a value that may either open a nested scope or decode as a + * scalar. Deeper lines open a nested object; inline values on a field + * that does not open a scope are rejected in strict mode (§14.2). + * + * @param value the value string to parse + * @param depth the depth at which the value is located + * @param context decode an object to deal with lines, delimiter and options + * @param scalarParser parses the value when it does not open a scope + * @return the parsed value (Map or scalar) + */ + static Object parseValueWithNestedScope(final String value, final int depth, final DecodeContext context, + final BiFunction scalarParser) { + // Check if the next line is nested (deeper indentation) if (context.currentLine + 1 < context.lines.length) { final int nextDepth = DecodeHelper.getDepth(context.lines[context.currentLine + 1], context); - if (nextDepth > fieldDepth) { - context.currentLine++; - // parseNestedObject manages the currentLine, so we don't increment here - return parseNestedObject(fieldDepth, context); - } else { - // If the value is empty, create an empty object; otherwise parse as primitive - if (fieldValue.isBlank()) { - context.currentLine++; - return new LinkedHashMap<>(); - } else { - context.currentLine++; - return PrimitiveDecoder.parse(fieldValue, context); + if (nextDepth > depth) { + if (!value.isBlank()) { + return parseInlineValueWithOrphanLines(value, depth, nextDepth, context, scalarParser); } - } - } else { - // If the value is empty, create an empty object; otherwise parse as primitive - if (fieldValue.isBlank()) { - context.currentLine++; - return new LinkedHashMap<>(); - } else { context.currentLine++; - return PrimitiveDecoder.parse(fieldValue, context); + // parseNestedObject manages the currentLine, so we don't increment here + return parseNestedObject(depth, context); } + context.currentLine++; + return scalarParser.apply(value, context); + } + context.currentLine++; + return scalarParser.apply(value, context); + } + + /** + * Parses an inline value whose line carries deeper, orphaned lines: + * rejected in strict mode (§14.2), skipped in non-strict mode. + * + * @param value the inline value string to parse + * @param depth the depth at which the value is located + * @param nextDepth the depth of the first orphaned line + * @param context decode an object to deal with lines, delimiter and options + * @param scalarParser parses the inline value + * @return the parsed scalar value + */ + private static Object parseInlineValueWithOrphanLines(final String value, final int depth, final int nextDepth, + final DecodeContext context, final BiFunction scalarParser) { + // Inline value: the field does not open a scope, so a deeper + // line belongs to no scope at all (§14.2) + if (context.options.strict()) { + throw new IllegalArgumentException( + "Over-indented line at " + (context.currentLine + 2) + " (depth " + nextDepth + ")"); } + // Non-strict: skip the orphaned lines and keep the inline value + do { + context.currentLine++; + } while (context.currentLine < context.lines.length + && DecodeHelper.getDepth(context.lines[context.currentLine], context) > depth); + return scalarParser.apply(value, context); } /** diff --git a/src/main/java/dev/toonformat/jtoon/decoder/PrimitiveDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/PrimitiveDecoder.java index fdba04a9..1b15ed6e 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/PrimitiveDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/PrimitiveDecoder.java @@ -1,10 +1,11 @@ package dev.toonformat.jtoon.decoder; import dev.toonformat.jtoon.util.StringEscaper; +import java.util.regex.Pattern; import static dev.toonformat.jtoon.util.Constants.DOT; +import static dev.toonformat.jtoon.util.Constants.FALSE_LITERAL; import static dev.toonformat.jtoon.util.Constants.NULL_LITERAL; import static dev.toonformat.jtoon.util.Constants.TRUE_LITERAL; -import static dev.toonformat.jtoon.util.Constants.FALSE_LITERAL; /** * Handles parsing of primitive TOON values with type inference. @@ -34,6 +35,12 @@ */ public final class PrimitiveDecoder { + // Normative number grammar of TOON spec §4: ^-?[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]+)?$ + // (case-insensitive). No leading '+': that is the wider encoder-side + // numeric-like test of §7.2. Tokens failing the gate (.5, 1., +1, NaN, 0x10) + // decode as strings without delegating to a host-language number parser. + private static final Pattern NUMBER_GRAMMAR = Pattern.compile("^-?\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$"); + private PrimitiveDecoder() { throw new UnsupportedOperationException("Utility class cannot be instantiated"); } @@ -83,50 +90,127 @@ static Object parse(final String value, final int maxStringLength) { // Check for quoted strings if (value.startsWith("\"")) { + // Spec §7.4: a quoted token is terminated by its closing quote; + // only whitespace may follow within the same token. The boundary + // rule applies in strict and non-strict mode alike. + validateQuotedTokenBoundary(value); // Validate string before unescaping StringEscaper.validateString(value); return StringEscaper.unescape(value); } - // Check for forbidden leading zeros (treat as string, except for "0", "-0", "0.0", etc.) - // Per spec §4: tokens like "05", "0001", "-05", "-0001" must be treated as strings. - // But "0.5", "0e1", "-0.5", "-0e1" are valid numbers. + return parseNumericToken(value); + } + + /** + * Parses an unquoted token as a number. Tokens failing the normative + * number grammar gate or carrying forbidden leading zeros decode as + * strings, without delegating to a host-language number parser (§4). + * + * @param value the token to parse + * @return the parsed number, or the token as string + */ + private static Object parseNumericToken(final String value) { final String trimmed = value.trim(); - if (trimmed.length() > 1) { - // Match forbidden leading zeros: starts with optional '-', then one or more zeros, - // then another digit (0-9) — meaning it's a multi-digit number with leading zeros. - // Exclude cases where the zero is part of a fractional/exponent form like "0.5", "0e1". - final boolean hasLeadingZeros = trimmed.matches("^-?0+\\d.*"); - // But we must NOT match "0.5" style numbers (single zero integer part) - final boolean isLikelyFractionalOrExponent = trimmed.matches("^-?0[.eE].*"); - if (hasLeadingZeros && !isLikelyFractionalOrExponent) { - return value; // treat as string - } + + // Normative number grammar gate (§4): tokens that do not match decode as + // strings, without delegating to a host-language number parser (§4). + if (!NUMBER_GRAMMAR.matcher(trimmed).matches()) { + return value; + } + + if (hasForbiddenLeadingZeros(trimmed)) { + return value; // treat as string } // Try parsing as number try { // Check if it contains exponent notation or decimal point - if (value.contains("e") || value.contains("E") || value.contains(DOT)) { - final double parsed = Double.parseDouble(value); - // Handle negative zero - Java doesn't distinguish, but spec says it should be 0 - if (parsed == 0.0) { - return 0L; - } - // Check if the result is a whole number - if so, return as Long - if (!Double.isInfinite(parsed) - && parsed >= Long.MIN_VALUE - && parsed <= Long.MAX_VALUE - && parsed == Math.floor(parsed)) { - return (long) parsed; - } - - return parsed; - } else { - return Long.parseLong(value); + if (isFloatingPointToken(value)) { + return normalizeFloatingPoint(Double.parseDouble(value)); } + return Long.parseLong(value); } catch (NumberFormatException e) { return value; } } + + /** + * Per spec §4: tokens like "05", "0001", "-05", "-0001" must be treated + * as strings. But "0.5", "0e1", "-0.5", "-0e1" are valid numbers. + * + * @param trimmed the trimmed token to check + * @return true when the token carries forbidden leading zeros + */ + private static boolean hasForbiddenLeadingZeros(final String trimmed) { + if (trimmed.length() <= 1) { + return false; + } + // Match forbidden leading zeros: starts with optional '-', then one or more zeros, + // then another digit (0-9) — meaning it's a multi-digit number with leading zeros. + // Exclude cases where the zero is part of a fractional/exponent form like "0.5", "0e1". + final boolean hasLeadingZeros = trimmed.matches("^-?0+\\d.*"); + // But we must NOT match "0.5" style numbers (single zero integer part) + final boolean isLikelyFractionalOrExponent = trimmed.matches("^-?0[.eE].*"); + return hasLeadingZeros && !isLikelyFractionalOrExponent; + } + + /** + * Returns whether the token contains exponent notation or a decimal point. + * + * @param value the token to check + * @return true for floating point tokens + */ + private static boolean isFloatingPointToken(final String value) { + return value.contains("e") || value.contains("E") || value.contains(DOT); + } + + /** + * Normalizes a parsed floating point value: negative zero collapses to + * zero, and whole numbers within long range are returned as Long. + * + * @param parsed the parsed double value + * @return the normalized number + */ + private static Object normalizeFloatingPoint(final double parsed) { + // Handle negative zero - Java doesn't distinguish, but spec says it should be 0 + if (parsed == 0.0) { + return 0L; + } + // Check if the result is a whole number - if so, return as Long + if (!Double.isInfinite(parsed) + && parsed >= Long.MIN_VALUE + && parsed <= Long.MAX_VALUE + && parsed == Math.floor(parsed)) { + return (long) parsed; + } + + return parsed; + } + + /** + * Spec §7.4: after the closing quote of a quoted token only whitespace may + * follow. An unterminated token is left to {@link StringEscaper#validateString}. + * + * @param value the token to validate + */ + private static void validateQuotedTokenBoundary(final String value) { + boolean escaped = false; + for (int i = 1; i < value.length(); i++) { + final char c = value.charAt(i); + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '"') { + for (int j = i + 1; j < value.length(); j++) { + if (!Character.isWhitespace(value.charAt(j))) { + throw new FatalDecodeException( + "Characters after closing quote in token: " + value); + } + } + return; + } + } + } } diff --git a/src/main/java/dev/toonformat/jtoon/decoder/TabularArrayDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/TabularArrayDecoder.java index 53562b0e..8f159af0 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/TabularArrayDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/TabularArrayDecoder.java @@ -4,9 +4,11 @@ import dev.toonformat.jtoon.util.StringEscaper; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import static dev.toonformat.jtoon.util.Constants.BACKSLASH; import static dev.toonformat.jtoon.util.Constants.DOUBLE_QUOTE; @@ -30,6 +32,17 @@ private TabularArrayDecoder() { throw new UnsupportedOperationException("Utility class cannot be instantiated"); } + /** + * One entry of a tabular header's field list (§6, §9.3). A leaf field + * carries an empty child list; a field with a nested field group carries + * its own ordered subfield list. + * + * @param name the field name + * @param children subfields of a nested field group, empty for a leaf field + */ + record FieldNode(String name, List children) { + } + /** * Parses tabular array format where each row contains delimiter-separated * values. @@ -49,7 +62,14 @@ public static List parseTabularArray(final String header, final int dept } final String keysStr = matcher.group(4); - final List keys = parseTabularKeys(keysStr, arrayDelimiter, context); + final List fields = parseTabularKeys(keysStr, arrayDelimiter, context); + + // Spec §9.3: a duplicate field name within one field list is a header + // defect, diagnosed from the header line alone. Names repeated at + // different nesting levels are not duplicates. + if (context.options.strict()) { + validateNoDuplicateFields(fields, context); + } final List result = new ArrayList<>(); context.currentLine++; @@ -64,39 +84,148 @@ public static List parseTabularArray(final String header, final int dept } while (context.currentLine < context.lines.length) { - if (!processTabularArrayLine(expectedRowDepth, keys, arrayDelimiter, result, context)) { + if (!processTabularArrayLine(expectedRowDepth, fields, arrayDelimiter, result, context)) { break; } } - ArrayDecoder.validateArrayLength(header, result.size(), context.options.maxArraySize()); + ArrayDecoder.validateArrayLength(header, result.size(), context.options.maxArraySize(), + context.options.strict()); return Collections.unmodifiableList(result); } /** * Parses tabular header keys from field specification. * Validates delimiter consistency between bracket and brace fields. + * Nested field groups ({@code field{sub1,sub2}}) become inner field nodes. * * @param keysStr the string representation of keys * @param arrayDelimiter the type of delimiter used in the array * @param context decode an object to deal with lines, delimiter and options - * @return list of keys + * @return the parsed field tree */ - private static List parseTabularKeys(final String keysStr, final Delimiter arrayDelimiter, + static List parseTabularKeys(final String keysStr, final Delimiter arrayDelimiter, final DecodeContext context) { // Validate delimiter mismatch between bracket and brace fields if (context.options.strict()) { validateKeysDelimiter(keysStr, arrayDelimiter); } - final List rawValues = ArrayDecoder.parseDelimitedValues(keysStr, arrayDelimiter); - final List result = new ArrayList<>(rawValues.size()); - for (final String key : rawValues) { - result.add(StringEscaper.unescape(key)); - } + final List result = new ArrayList<>(); + parseFieldList(keysStr, 0, arrayDelimiter, context, result); return result; } + /** + * Recursively parses a field list. Braces outside quoted names open a + * nested field group parsed with the same delimiter (§6, §9.3). + * + * @param fieldList the field list string to parse + * @param start the index at which parsing starts + * @param arrayDelimiter the type of delimiter used in the array + * @param context decode an object to deal with lines, delimiter and options + * @param result the list to add parsed fields to + * @return the index just past the closing brace of the parsed group, or -1 + * when the string ends before a group is closed + */ + private static int parseFieldList(final String fieldList, final int start, final Delimiter arrayDelimiter, + final DecodeContext context, final List result) { + final char delimiterChar = arrayDelimiter.toString().charAt(0); + final StringBuilder name = new StringBuilder(); + boolean inQuotes = false; + boolean escaped = false; + int i = start; + while (i < fieldList.length()) { + final char c = fieldList.charAt(i); + if (escaped) { + name.append(c); + escaped = false; + i++; + } else if (c == BACKSLASH) { + name.append(c); + escaped = true; + i++; + } else if (c == DOUBLE_QUOTE) { + name.append(c); + inQuotes = !inQuotes; + i++; + } else if (!inQuotes && c == '{') { + i = parseNestedFieldGroup(fieldList, i, arrayDelimiter, context, result, name); + } else if (!inQuotes && c == '}') { + flushField(result, name); + return i + 1; + } else if (!inQuotes && c == delimiterChar) { + i = skipFieldDelimiter(fieldList, i, result, name); + } else { + name.append(c); + i++; + } + } + flushField(result, name); + return -1; + } + + /** + * Parses a nested field group opened at the given brace, recursing into + * {@link #parseFieldList}. Unbalanced groups are rejected in strict mode + * and skipped in lenient mode. + * + * @param fieldList the field list string to parse + * @param braceIdx the index of the opening brace + * @param arrayDelimiter the type of delimiter used in the array + * @param context decode an object to deal with lines, delimiter and options + * @param result the list to add the parsed group field to + * @param name the buffered group field name + * @return the index just past the closing brace, or the end of the string + * when the group is unbalanced and lenient mode skips it + */ + private static int parseNestedFieldGroup(final String fieldList, final int braceIdx, + final Delimiter arrayDelimiter, final DecodeContext context, final List result, + final StringBuilder name) { + final List children = new ArrayList<>(); + final int next = parseFieldList(fieldList, braceIdx + 1, arrayDelimiter, context, children); + if (next < 0) { + if (context.options.strict()) { + throw new IllegalArgumentException( + "Unbalanced braces in tabular header field list"); + } + return fieldList.length(); + } + result.add(new FieldNode(StringEscaper.unescape(name.toString().trim()), children)); + name.setLength(0); + return next; + } + + /** + * Flushes the buffered field and skips the delimiter together with any + * following whitespace. + * + * @param fieldList the field list string to parse + * @param delimiterIdx the index of the delimiter character + * @param result the list to add the flushed field to + * @param name the buffered field name + * @return the index just past the delimiter and trailing whitespace + */ + private static int skipFieldDelimiter(final String fieldList, final int delimiterIdx, + final List result, final StringBuilder name) { + flushField(result, name); + int i = delimiterIdx + 1; + while (i < fieldList.length() && Character.isWhitespace(fieldList.charAt(i))) { + i++; + } + return i; + } + + /** + * Adds the buffered field name as a leaf node and resets the buffer. + */ + private static void flushField(final List result, final StringBuilder name) { + if (!name.isEmpty()) { + result.add(new FieldNode(StringEscaper.unescape(name.toString().trim()), Collections.emptyList())); + name.setLength(0); + } + } + /** * Validates delimiter consistency in tabular header keys. * @@ -148,18 +277,24 @@ private static void checkDelimiterMismatch(final char expectedChar, final char a * Processes a single line in a tabular array. * * @param expectedRowDepth the expected depth of the next row - * @param keys the keys for the tabular array + * @param fields the field tree for the tabular array * @param arrayDelimiter the type of delimiter used in the array * @param result the list to store parsed rows in * @param context decode an object to deal with lines, delimiter and options * @return true if parsing should continue, false if an array should terminate */ - private static boolean processTabularArrayLine(final int expectedRowDepth, final List keys, + private static boolean processTabularArrayLine(final int expectedRowDepth, final List fields, final Delimiter arrayDelimiter, final List result, final DecodeContext context) { final String line = context.lines[context.currentLine]; if (DecodeHelper.isBlankLine(line)) { + // Spec §12: blank lines between the header and the first row are + // accepted even in strict mode + if (result.isEmpty()) { + context.currentLine++; + return true; + } return !handleBlankLineInTabularArray(expectedRowDepth, context); } @@ -168,7 +303,7 @@ private static boolean processTabularArrayLine(final int expectedRowDepth, final return false; } - if (processTabularRow(line, lineDepth, expectedRowDepth, keys, arrayDelimiter, result, context)) { + if (processTabularRow(line, lineDepth, expectedRowDepth, fields, arrayDelimiter, result, context)) { context.currentLine++; } return true; @@ -184,13 +319,15 @@ private static boolean processTabularArrayLine(final int expectedRowDepth, final private static boolean handleBlankLineInTabularArray(final int expectedRowDepth, final DecodeContext context) { final int nextNonBlankLine = DecodeHelper.findNextNonBlankLine(context.currentLine + 1, context); - if (nextNonBlankLine < context.lines.length) { - final int nextDepth = DecodeHelper.getDepth(context.lines[nextNonBlankLine], context); - // Header depth is one level above the expected row depth - final int headerDepth = expectedRowDepth - 1; - if (nextDepth <= headerDepth) { - return true; - } + if (nextNonBlankLine >= context.lines.length) { + // Blank lines at the end of the document are trailing newlines (§12) + return true; + } + final int nextDepth = DecodeHelper.getDepth(context.lines[nextNonBlankLine], context); + // Header depth is one level above the expected row depth + final int headerDepth = expectedRowDepth - 1; + if (nextDepth <= headerDepth) { + return true; } // Blank line is inside the array @@ -283,22 +420,27 @@ private static int findFirstUnquoted(final String content, final char target) { * @param line the line to process * @param lineDepth the depth of the line * @param expectedRowDepth the expected depth of the next row - * @param keys the keys for the tabular array + * @param fields the field tree for the tabular array * @param arrayDelimiter the type of delimiter used in the array * @param result the list to store parsed rows in * @param context decode an object to deal with lines, delimiter and options * @return true if a line was processed and the currentLine should be incremented, false otherwise. */ private static boolean processTabularRow(final String line, final int lineDepth, - final int expectedRowDepth, final List keys, final Delimiter arrayDelimiter, + final int expectedRowDepth, final List fields, final Delimiter arrayDelimiter, final List result, final DecodeContext context) { if (lineDepth == expectedRowDepth) { final String rowContent = line.substring(expectedRowDepth * context.options.indent()); - final Map row = parseTabularRow(rowContent, keys, arrayDelimiter, context); + final Map row = parseTabularRow(rowContent, fields, arrayDelimiter, context); result.add(row); return true; } else if (lineDepth > expectedRowDepth) { - // Line is deeper than expected - might be nested content, skip it + // A line deeper than the row depth belongs to no scope (§14.2) + if (context.options.strict()) { + throw new IllegalArgumentException( + "Over-indented line after tabular rows at line " + (context.currentLine + 1)); + } + // In non-strict mode, skip it context.currentLine++; return false; } @@ -306,36 +448,86 @@ private static boolean processTabularRow(final String line, final int lineDepth, } /** - * Parses a tabular row into a Map using the provided keys. - * Validates that the row uses the correct delimiter. + * Parses a tabular row into a Map using the provided field tree. + * A leaf field consumes the next cell; a nested field group materializes + * an object from its subfields (§9.3). * - *

In strict mode, the number of values must exactly match the number of keys. - * In lenient mode, excess values are silently dropped and missing values - * result in omitted keys.

+ *

In strict mode, the number of values must exactly match the leaf-field + * count. In lenient mode, excess values are silently dropped and missing + * values result in omitted keys.

* * @param rowContent the row content to parse - * @param keys the keys for the tabular array + * @param fields the field tree for the tabular array * @param arrayDelimiter the type of delimiter used in the array * @param context decode an object to deal with lines, delimiter and options * @return a Map containing the parsed row values */ - private static Map parseTabularRow(final String rowContent, final List keys, - final Delimiter arrayDelimiter, final DecodeContext context) { + static Map parseTabularRow(final String rowContent, final List fields, + final Delimiter arrayDelimiter, final DecodeContext context) { final Map row = new LinkedHashMap<>(); final List values = ArrayDecoder.parseArrayValues(rowContent, arrayDelimiter, context.options.maxArraySize(), context.options.maxStringLength()); - // Validate value count matches key count - if (context.options.strict() && values.size() != keys.size()) { + // Spec §9.3: each row must carry exactly one cell per leaf field + if (context.options.strict() && values.size() != countLeaves(fields)) { throw new IllegalArgumentException( - String.format("Tabular row value count (%d) does not match header field count (%d)", - values.size(), keys.size())); + String.format("Tabular row value count (%d) does not match header leaf-field count (%d)", + values.size(), countLeaves(fields))); } - for (int i = 0; i < keys.size() && i < values.size(); i++) { - row.put(keys.get(i), values.get(i)); - } + assignRowValues(fields, values, row, 0); return row; } + + /** + * Assigns row cells to the field tree in depth-first, pre-order walk + * order (§9.3): a leaf field takes the next cell, a nested field group + * materializes an object from its subfields. + */ + static void assignRowValues(final List fields, final List values, + final Map target, final int... nextCell) { + for (final FieldNode field : fields) { + if (field.children().isEmpty()) { + final int index = nextCell[0]; + nextCell[0] = index + 1; + if (index < values.size()) { + target.put(field.name(), values.get(index)); + } + } else { + final Map group = new LinkedHashMap<>(); + assignRowValues(field.children(), values, group, nextCell); + target.put(field.name(), group); + } + } + } + + /** + * Counts the leaf fields of a field tree (§9.3): each row carries exactly + * one cell per leaf field. + */ + static int countLeaves(final List fields) { + int count = 0; + for (final FieldNode field : fields) { + count += field.children().isEmpty() ? 1 : countLeaves(field.children()); + } + return count; + } + + /** + * Spec §9.3: a duplicate field name within one field list is a header + * defect, checked recursively at every nesting level. + */ + static void validateNoDuplicateFields(final List fields, final DecodeContext context) { + final Set seen = new HashSet<>(fields.size()); + for (final FieldNode field : fields) { + if (!seen.add(field.name())) { + throw new IllegalArgumentException( + "Duplicate field name '" + field.name() + "' in tabular header"); + } + if (!field.children().isEmpty()) { + validateNoDuplicateFields(field.children(), context); + } + } + } } diff --git a/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java b/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java index 3d633213..8530a718 100644 --- a/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java +++ b/src/main/java/dev/toonformat/jtoon/decoder/ValueDecoder.java @@ -1,14 +1,15 @@ package dev.toonformat.jtoon.decoder; import dev.toonformat.jtoon.DecodeOptions; +import dev.toonformat.jtoon.util.Headers; import dev.toonformat.jtoon.util.ObjectMapperSingleton; import org.jspecify.annotations.Nullable; import tools.jackson.databind.ObjectMapper; +import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.regex.Matcher; +import java.util.List; import static dev.toonformat.jtoon.util.Constants.NULL_LITERAL; import static dev.toonformat.jtoon.util.Constants.OPEN_BRACKET; -import static dev.toonformat.jtoon.util.Headers.KEYED_ARRAY_PATTERN; /** * Main decoder for converting TOON-formatted strings to Java objects. @@ -32,6 +33,7 @@ public final class ValueDecoder { private static final ObjectMapper MAPPER = ObjectMapperSingleton.getInstance(); + private static final int BOM_CHARACTER = 0xFEFF; private ValueDecoder() { throw new UnsupportedOperationException("Utility class cannot be instantiated"); @@ -50,6 +52,11 @@ private ValueDecoder() { public static Object decode(final String toon, final DecodeOptions options) { try { return decodeInternal(toon, options); + } catch (FatalDecodeException e) { + // Spec §5.2/§7.4: bare scalars outside root primitive position and + // characters after a closing quote are errors in strict and + // non-strict mode alike; lenient mode must not swallow them. + throw e; } catch (IllegalArgumentException e) { if (!options.strict()) { return null; @@ -60,12 +67,19 @@ public static Object decode(final String toon, final DecodeOptions options) { @Nullable private static Object decodeInternal(final String toon, final DecodeOptions options) { - if (toon == null || toon.isBlank()) { + if (toon == null) { + return new LinkedHashMap<>(); + } + + // Spec §5.1: a single U+FEFF at the very start of the document is a + // byte-order mark, not content; remove it before any processing. + final String input = stripByteOrderMark(toon); + + if (input.isBlank()) { return new LinkedHashMap<>(); } - // Special case: if input is exactly "null", return null - final String trimmed = toon.trim(); + final String trimmed = input.trim(); if (NULL_LITERAL.equals(trimmed)) { return null; } @@ -73,18 +87,17 @@ private static Object decodeInternal(final String toon, final DecodeOptions opti return java.util.Collections.emptyList(); } - // Don't trim leading whitespace - we need it for indentation validation - // Only trim trailing whitespace to avoid issues with empty lines at the end - final String processed = Character.isWhitespace(toon.charAt(toon.length() - 1)) - ? toon.stripTrailing() - : toon; - //set an own decode context final DecodeContext context = new DecodeContext(); - context.lines = processed.split("\r?\n", -1); + context.lines = buildContentLines(input.split("\r?\n", -1)); context.options = options; context.delimiter = options.delimiter(); + // Spec §5.1: a document of only comments and blank lines is an empty object + if (isEmptyDocument(context.lines)) { + return new LinkedHashMap<>(); + } + final int lineIndex = context.currentLine; final String line = context.lines[lineIndex]; final int depth = DecodeHelper.getDepth(line, context); @@ -96,46 +109,108 @@ private static Object decodeInternal(final String toon, final DecodeOptions opti return new LinkedHashMap<>(); } - // Handle standalone arrays: [2]: - if (!line.isEmpty() && line.charAt(0) == OPEN_BRACKET.charAt(0)) { - return ArrayDecoder.parseArray(line, depth, context); - } - - // Handle keyed arrays: items[2]{id,name}: - // Only match if the key part (before the bracket) doesn't contain a colon, - // because a colon indicates a key-value pair (e.g. 'a: "[2]: x"') - final Matcher keyedArray = KEYED_ARRAY_PATTERN.matcher(line); - if (keyedArray.matches()) { - final String keyPart = keyedArray.group(1); - final int colonInKey = DecodeHelper.findUnquotedColon(keyPart); - if (colonInKey <= 0) { - return KeyDecoder.parseKeyedArrayValue(keyedArray, line, depth, context); + final Object result = parseRootDocument(line, depth, context); + + // The root form spans the whole document (§5); leftover lines must not be + // silently discarded. + DecodeHelper.validateNoTrailingContent(context); + return result; + } + + private static String stripByteOrderMark(final String input) { + if (!input.isEmpty() && input.charAt(0) == BOM_CHARACTER) { + return input.substring(1); + } + return input; + } + + /** + * Builds the list of content lines: trailing spaces are stripped per line + * (§12) and full-line comments are discarded (§5.1). + */ + private static String[] buildContentLines(final String... rawLines) { + // Spec §12: trailing spaces at the end of a line are not part of its content; + // strip them per line before classification. Only characters after the last + // non-space character are removed, so trailing spaces inside quoted strings + // (e.g. key: "a ") are preserved. + // Spec §5.1: a line whose first non-space character (U+0020 only) is '#' + // is a full-line comment; it is discarded before any structural + // interpretation. A tab before '#' disqualifies the line, and a '#' + // anywhere else is data, not a comment. + final List contentLines = new ArrayList<>(rawLines.length); + for (String rawLine : rawLines) { + final String stripped = rawLine.stripTrailing(); + if (!isCommentLine(stripped)) { + contentLines.add(stripped); } } - // Handle key-value pairs: name: Ada + return contentLines.toArray(new String[0]); + } + + private static boolean isEmptyDocument(final String... lines) { + for (final String line : lines) { + if (!line.isBlank()) { + return false; + } + } + return true; + } + + /** + * Routes the root line to its form (§5): keyless array header, keyed + * array header, key-value pair, or bare scalar. + */ + private static Object parseRootDocument(final String line, final int depth, final DecodeContext context) { + if (!line.isEmpty() && line.charAt(0) == OPEN_BRACKET.charAt(0)) { + return parseRootArrayLine(line, depth, context); + } + + final Headers.KeyedHeaderMatch keyedHeader = Headers.matchKeyedArrayHeader(line); + if (keyedHeader != null) { + return KeyDecoder.parseKeyedArrayValue(keyedHeader, line, depth, context); + } + final int colonIdx = DecodeHelper.findUnquotedColon(line); if (colonIdx > 0) { - if (context.options.strict()) { - final String key = line.substring(0, colonIdx).trim(); - // In strict mode, reject keys with unquoted brackets that didn't match - // KEYED_ARRAY_PATTERN. This catches: - // - extra brackets between bracket segment and colon (foo[1][bar]) - // - text between bracket segment and colon (foo[2]extra) - // - non-integer bracket segment (foo[bar]) - // - negative bracket length (items[-1]) - // - whitespace between bracket segment and colon/fields segment - // (items[2] :, items[2] {a,b}:) - if (DecodeHelper.hasUnquotedBrackets(key)) { - throw new IllegalArgumentException( - "Invalid array header syntax at line " + (context.currentLine + 1)); - } - } + return parseRootKeyValueLine(line, colonIdx, depth, context); + } + + return parseRootBareLine(line, depth, context); + } + + private static Object parseRootArrayLine(final String line, final int depth, final DecodeContext context) { + // Spec §5/§9.5: a keyed marker in the bracket segment makes a + // keyless header a keyed tabular object, not an array. + final Headers.KeyedHeaderMatch keylessHeader = Headers.matchKeylessKeyedHeader(line); + if (keylessHeader != null && keylessHeader.keyed()) { + return KeyedObjectDecoder.parseKeyedTabularObject(line, keylessHeader, depth + 1, context); + } + return ArrayDecoder.parseArray(line, depth, context); + } + + private static Object parseRootKeyValueLine(final String line, final int colonIdx, final int depth, + final DecodeContext context) { + if (context.options.strict()) { final String key = line.substring(0, colonIdx).trim(); - final String value = line.substring(colonIdx + 1).trim(); - return KeyDecoder.parseKeyValuePair(key, value, depth, depth == 0, context); + // In strict mode, reject keys with unquoted brackets that didn't match + // KEYED_ARRAY_PATTERN. This catches: + // - extra brackets between bracket segment and colon (foo[1][bar]) + // - text between bracket segment and colon (foo[2]extra) + // - noninteger bracket segment (foo[bar]) + // - negative bracket length (items[-1]) + // - whitespace between bracket segment and colon/fields segment + // (items[2] :, items[2] {a,b}:) + if (DecodeHelper.hasUnquotedBrackets(key)) { + throw new IllegalArgumentException( + "Invalid array header syntax at line " + (context.currentLine + 1)); + } } + final String key = line.substring(0, colonIdx).trim(); + final String value = line.substring(colonIdx + 1).trim(); + return KeyDecoder.parseKeyValuePair(key, value, depth, depth == 0, context); + } - // Bare scalar value + private static Object parseRootBareLine(final String line, final int depth, final DecodeContext context) { if (context.options.strict() && DecodeHelper.hasUnquotedBrackets(line)) { // Line has brackets but no colon and didn't match KEYED_ARRAY_PATTERN // (e.g. "items[2]{id,name}" missing colon) @@ -146,6 +221,22 @@ private static Object decodeInternal(final String toon, final DecodeOptions opti return ObjectDecoder.parseBareScalarValue(line, depth, context); } + /** + * Spec §5.1: a full-line comment is a line whose first non-space character + * is '#'. Only U+0020 spaces may precede it; a tab or any other character + * disqualifies the line, and '#' anywhere else is data. + * + * @param line the stripped line to test + * @return true if the line is a full-line comment + */ + private static boolean isCommentLine(final String line) { + int index = 0; + while (index < line.length() && line.charAt(index) == ' ') { + index++; + } + return index < line.length() && line.charAt(index) == '#'; + } + /** * Decodes a TOON-formatted string directly to a JSON string using custom * options. diff --git a/src/main/java/dev/toonformat/jtoon/encoder/ArrayEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/ArrayEncoder.java index 166856f3..3708454b 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/ArrayEncoder.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/ArrayEncoder.java @@ -31,75 +31,114 @@ private ArrayEncoder() { */ public static void encodeArray(@Nullable final String key, final ArrayNode value, final LineWriter writer, final int depth, final EncodeOptions options) { - if (value.isEmpty()) { - // Per spec §9.1: encoders SHOULD emit key: [] for empty arrays. - // When lengthMarker is enabled, use the legacy header form instead. - if (key == null && depth == 0) { - writer.push(depth, options.lengthMarker() ? "[0]: " : "[]"); - return; - } - if (key != null && !options.lengthMarker()) { - final String encodedKey = PrimitiveEncoder.encodeKey(key); - writer.push(depth, encodedKey + ": []"); - return; - } - final String header = PrimitiveEncoder.formatHeader(0, key, null, options.delimiter().toString(), - options.lengthMarker()); - writer.push(depth, header); - return; + final ArrayShape shape = classifyContents(value); + switch (shape) { + case EMPTY -> encodeEmptyArray(key, writer, depth, options); + case PRIMITIVES -> encodeInlinePrimitiveArray(key, value, writer, depth, options); + case ARRAYS_OF_PRIMITIVES -> encodeArrayOfArraysAsListItems(key, value, writer, depth, options); + case OBJECTS -> encodeObjectArray(key, value, writer, depth, options); + default -> encodeMixedArrayAsListItems(key, value, writer, depth, options); } + } - final int size = value.size(); - boolean allPrimitives = true; - boolean allArrays = true; - boolean allObjects = true; + /** + * Categorizes the homogeneous shape of an array: empty, all primitives, + * all primitive arrays, all objects, or mixed content. + */ + private enum ArrayShape { + EMPTY, + PRIMITIVES, + ARRAYS_OF_PRIMITIVES, + OBJECTS, + MIXED + } - for (int i = 0; i < size; i++) { - final JsonNode item = value.get(i); - if (!item.isValueNode()) { - allPrimitives = false; - } - if (!item.isArray()) { - allArrays = false; - } - if (!item.isObject()) { - allObjects = false; - } - if (!allPrimitives && !allArrays && !allObjects) { - break; - } - } + /** + * Tracks the homogeneity flags observed across all items. + * + * @param allPrimitives whether every item is a primitive value + * @param allArrays whether every item is an array + * @param allObjects whether every item is an object + * @param allPrimitiveArrays whether every item is an array of primitives + */ + private record ShapeFlags(boolean allPrimitives, boolean allArrays, boolean allObjects, + boolean allPrimitiveArrays) { + } - if (allPrimitives) { - encodeInlinePrimitiveArray(key, value, writer, depth, options); - return; + /** + * Classifies the array contents into the homogeneous shape with the + * most specific encoding strategy. + * + * @param value the array to classify + * @return the detected shape + */ + private static ArrayShape classifyContents(final ArrayNode value) { + if (value.isEmpty()) { + return ArrayShape.EMPTY; } - - if (allArrays) { - boolean allPrimitiveArrays = true; - for (int i = 0; i < size; i++) { - if (!isArrayOfPrimitives(value.get(i))) { - allPrimitiveArrays = false; - break; - } - } - if (allPrimitiveArrays) { - encodeArrayOfArraysAsListItems(key, value, writer, depth, options); - return; - } + ShapeFlags flags = new ShapeFlags(true, true, true, true); + for (int i = 0; i < value.size(); i++) { + flags = updateShape(flags, value.get(i)); + } + if (flags.allPrimitives()) { + return ArrayShape.PRIMITIVES; } + if (flags.allArrays() && flags.allPrimitiveArrays()) { + return ArrayShape.ARRAYS_OF_PRIMITIVES; + } + if (flags.allObjects()) { + return ArrayShape.OBJECTS; + } + return ArrayShape.MIXED; + } - if (allObjects) { - final List header = TabularArrayEncoder.detectTabularHeader(value); - if (!header.isEmpty()) { - TabularArrayEncoder.encodeArrayOfObjectsAsTabular(key, value, header, writer, depth, options); - } else { - encodeMixedArrayAsListItems(key, value, writer, depth, options); - } + /** + * Refines the homogeneity flags with a single item. + * + * @param flags the flags so far + * @param item the item to fold in + * @return the refined flags + */ + private static ShapeFlags updateShape(final ShapeFlags flags, final JsonNode item) { + return new ShapeFlags( + flags.allPrimitives() && item.isValueNode(), + flags.allArrays() && item.isArray(), + flags.allObjects() && item.isObject(), + flags.allPrimitiveArrays() && isArrayOfPrimitives(item)); + } + + /** + * Encodes an empty array per spec §9.1: emitters SHOULD use key: []. + * With the length marker enabled, the legacy header form is used instead. + */ + private static void encodeEmptyArray(@Nullable final String key, + final LineWriter writer, final int depth, final EncodeOptions options) { + if (key == null && depth == 0) { + writer.push(depth, options.lengthMarker() ? "[0]: " : "[]"); return; } + if (key != null && !options.lengthMarker()) { + final String encodedKey = PrimitiveEncoder.encodeKey(key); + writer.push(depth, encodedKey + ": []"); + return; + } + final String header = PrimitiveEncoder.formatHeader(0, key, null, options.delimiter().toString(), + options.lengthMarker()); + writer.push(depth, header); + } - encodeMixedArrayAsListItems(key, value, writer, depth, options); + /** + * Encodes an array of objects: tabular when a uniform header is + * detected, otherwise as list items. + */ + private static void encodeObjectArray(@Nullable final String key, final ArrayNode value, + final LineWriter writer, final int depth, final EncodeOptions options) { + final List header = TabularArrayEncoder.detectTabularHeader(value); + if (header.isEmpty()) { + encodeMixedArrayAsListItems(key, value, writer, depth, options); + } else { + TabularArrayEncoder.encodeArrayOfObjectsAsTabular(key, value, header, writer, depth, options); + } } /** diff --git a/src/main/java/dev/toonformat/jtoon/encoder/HeaderFormatter.java b/src/main/java/dev/toonformat/jtoon/encoder/HeaderFormatter.java index 5978fa7f..d6b90ef4 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/HeaderFormatter.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/HeaderFormatter.java @@ -25,14 +25,14 @@ private HeaderFormatter() { * * @param length Array or table length * @param key Optional key prefix - * @param fields Optional field names for tabular format + * @param fields Optional header fields for tabular format * @param delimiter The delimiter being used * @param lengthMarker Whether to include # marker before length */ public record HeaderConfig( int length, @Nullable String key, - @Nullable List fields, + @Nullable List fields, String delimiter, boolean lengthMarker) { } @@ -59,7 +59,7 @@ static String format(final HeaderConfig config) { * Delegates to the record-based format method. * @param length the array or table length * @param key optional key prefix - * @param fields optional field names for tabular format + * @param fields optional header fields for tabular format * @param delimiter the delimiter being used * @param lengthMarker whether to include # marker before length * @return formatted header string @@ -67,13 +67,48 @@ static String format(final HeaderConfig config) { public static String format( final int length, @Nullable final String key, - @Nullable final List fields, + @Nullable final List fields, final String delimiter, final boolean lengthMarker) { final HeaderConfig config = new HeaderConfig(length, key, fields, delimiter, lengthMarker); return format(config); } + /** + * Formats a keyed tabular header (§9.5): {@code key[N:]{fields}:}. + * The keyed marker colon is written directly after the entry count; a + * non-default delimiter follows it inside the brackets. + * + * @param count entry count + * @param key optional key prefix (omitted for the root form) + * @param fields header fields for keyed tabular form + * @param delimiter the delimiter being used + * @param lengthMarker whether to include # marker before the count + * @return formatted keyed header string + */ + public static String formatKeyedHeader( + final int count, + @Nullable final String key, + @Nullable final List fields, + final String delimiter, + final boolean lengthMarker) { + final StringBuilder header = new StringBuilder(); + + appendKeyIfPresent(header, key); + header.append(OPEN_BRACKET); + if (lengthMarker) { + header.append(HASHTAG); + } + header.append(count); + header.append(COLON); + appendDelimiterIfNotDefault(header, delimiter); + header.append(CLOSE_BRACKET); + appendFieldsIfPresent(header, fields, delimiter); + header.append(COLON); + + return header.toString(); + } + private static void appendKeyIfPresent(final StringBuilder header, @Nullable final String key) { if (key != null) { header.append(PrimitiveEncoder.encodeKey(key)); @@ -104,7 +139,7 @@ private static void appendDelimiterIfNotDefault(final StringBuilder header, fina private static void appendFieldsIfPresent( final StringBuilder header, - @Nullable final Collection fields, + @Nullable final Collection fields, final String delimiter) { if (fields == null || fields.isEmpty()) { return; @@ -115,19 +150,20 @@ private static void appendFieldsIfPresent( header.append(CLOSE_BRACE); } - private static String formatFields(final Collection fields, final String delimiter) { - if (fields.isEmpty()) { - return ""; - } - + private static String formatFields(final Collection fields, final String delimiter) { final StringBuilder sb = new StringBuilder(); boolean first = true; - for (final String field : fields) { + for (final TabularField field : fields) { if (!first) { sb.append(delimiter); } first = false; - sb.append(PrimitiveEncoder.encodeKey(field)); + sb.append(PrimitiveEncoder.encodeKey(field.name())); + if (!field.isLeaf()) { + sb.append(OPEN_BRACE); + sb.append(formatFields(field.children(), delimiter)); + sb.append(CLOSE_BRACE); + } } return sb.toString(); } diff --git a/src/main/java/dev/toonformat/jtoon/encoder/KeyedObjectEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/KeyedObjectEncoder.java new file mode 100644 index 00000000..8445baaa --- /dev/null +++ b/src/main/java/dev/toonformat/jtoon/encoder/KeyedObjectEncoder.java @@ -0,0 +1,106 @@ +package dev.toonformat.jtoon.encoder; + +import dev.toonformat.jtoon.EncodeOptions; +import org.jspecify.annotations.Nullable; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ObjectNode; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import static dev.toonformat.jtoon.util.Constants.COLON; +import static dev.toonformat.jtoon.util.Constants.SPACE; + +/** + * Detects and encodes objects of uniform objects in keyed tabular form (§9.5). + * The shared field structure is declared once in a keyed header; each entry + * becomes one row carrying its own key. + */ +public final class KeyedObjectEncoder { + + private KeyedObjectEncoder() { + throw new UnsupportedOperationException("Utility class cannot be instantiated"); + } + + /** + * Detects if an object can be encoded in keyed tabular form (§9.5): + * at least two entries, every entry value a non-empty object with the same + * key set, and every column uniform-primitive or nested-uniform (§9.3). + * Returns the header fields (ordered by the first entry's encounter order) + * or an empty list when the object must stay in nested form. + * + * @param entries the object whose values are candidate entry objects + * @return keyed header fields, or empty list if not keyed-eligible + */ + public static List detectKeyedFields(final ObjectNode entries) { + if (entries.size() < 2) { + return Collections.emptyList(); + } + + for (final JsonNode value : entries) { + if (!value.isObject() || value.isEmpty()) { + return Collections.emptyList(); + } + } + + final String firstKey = entries.propertyNames().iterator().next(); + final ObjectNode firstEntry = (ObjectNode) entries.get(firstKey); + final List header = new ArrayList<>(); + for (final String key : firstEntry.propertyNames()) { + final Optional> children = TabularArrayEncoder.uniformColumnsOf(firstEntry.get(key)); + if (children.isEmpty()) { + return Collections.emptyList(); + } + header.add(new TabularField(key, children.get())); + } + + if (!TabularArrayEncoder.matchesEveryRow(entries, header)) { + return Collections.emptyList(); + } + + return Collections.unmodifiableList(header); + } + + /** + * Encodes an object of uniform objects as a keyed table. + * + * @param prefix optional key prefix (null for the root keyless form) + * @param entries the object of uniform entry objects + * @param fields detected keyed header fields + * @param writer LineWriter for output + * @param depth Indentation depth + * @param options Encoding options + */ + public static void encodeKeyedTabularObject(@Nullable final String prefix, final ObjectNode entries, + final List fields, final LineWriter writer, final int depth, + final EncodeOptions options) { + final String headerStr = HeaderFormatter.formatKeyedHeader(entries.size(), prefix, fields, + options.delimiter().toString(), options.lengthMarker()); + writer.push(depth, headerStr); + + writeKeyedRows(entries, fields, writer, depth + 1, options); + } + + /** + * Writes entry rows: {@code entrykey: c1c2…} in entry + * encounter order (§9.5). The entry key is encoded per §7.3 and followed + * by a colon and a single space (§12). + * + * @param entries the object of uniform entry objects + * @param fields keyed header fields + * @param writer LineWriter for output + * @param depth Indentation depth + * @param options Encoding options + */ + public static void writeKeyedRows(final ObjectNode entries, final List fields, + final LineWriter writer, final int depth, final EncodeOptions options) { + final String delimiter = options.delimiter().toString(); + for (final String entryKey : entries.propertyNames()) { + final ObjectNode entryValue = (ObjectNode) entries.get(entryKey); + final List cells = new ArrayList<>(fields.size()); + TabularArrayEncoder.collectCells(entryValue, fields, cells, delimiter); + writer.push(depth, + PrimitiveEncoder.encodeKey(entryKey) + COLON + SPACE + String.join(delimiter, cells)); + } + } +} diff --git a/src/main/java/dev/toonformat/jtoon/encoder/ListItemEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/ListItemEncoder.java index 9a1bf75b..e35a6c97 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/ListItemEncoder.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/ListItemEncoder.java @@ -75,7 +75,7 @@ private static void encodeFirstKeyValue(final String key, } else if (value.isArray()) { encodeFirstValueAsArray(key, encodedKey, (ArrayNode) value, writer, depth, options); } else if (value.isObject()) { - encodeFirstValueAsObject(encodedKey, (ObjectNode) value, writer, depth, options); + encodeFirstValueAsObject(key, encodedKey, (ObjectNode) value, writer, depth, options); } } @@ -119,7 +119,7 @@ private static void encodeFirstArrayAsObjects(final String key, final LineWriter writer, final int depth, final EncodeOptions options) { - final List header = TabularArrayEncoder.detectTabularHeader(arrayValue); + final List header = TabularArrayEncoder.detectTabularHeader(arrayValue); if (!header.isEmpty()) { final String headerStr = PrimitiveEncoder.formatHeader(arrayValue.size(), key, header, options.delimiter().toString(), @@ -159,11 +159,20 @@ private static void encodeFirstArrayAsComplex(final String encodedKey, } } - private static void encodeFirstValueAsObject(final String encodedKey, + private static void encodeFirstValueAsObject(final String key, + final String encodedKey, final ObjectNode nestedObj, final LineWriter writer, final int depth, final EncodeOptions options) { + final List keyedFields = KeyedObjectEncoder.detectKeyedFields(nestedObj); + if (!keyedFields.isEmpty()) { + final String headerStr = HeaderFormatter.formatKeyedHeader(nestedObj.size(), key, keyedFields, + options.delimiter().toString(), options.lengthMarker()); + writer.push(depth, LIST_ITEM_PREFIX + headerStr); + KeyedObjectEncoder.writeKeyedRows(nestedObj, keyedFields, writer, depth + 2, options); + return; + } writer.push(depth, LIST_ITEM_PREFIX + encodedKey + COLON); if (!nestedObj.isEmpty()) { ObjectEncoder.encodeObject(nestedObj, writer, depth + 2, options, Set.of(), null, null, new HashSet<>()); diff --git a/src/main/java/dev/toonformat/jtoon/encoder/ObjectEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/ObjectEncoder.java index 00d58502..1aca7637 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/ObjectEncoder.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/ObjectEncoder.java @@ -7,6 +7,7 @@ import tools.jackson.databind.node.ArrayNode; import tools.jackson.databind.node.ObjectNode; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import static dev.toonformat.jtoon.util.Constants.DOT; @@ -105,34 +106,81 @@ public static void encodeKeyValuePair(final String key, final int remainingDepth = effectiveFlattenDepth - depth; EncodeOptions currentOptions = options; - if (remainingDepth > 0 + if (shouldTryFlatten(key, siblings, blockedKeys, remainingDepth, currentOptions)) { + currentOptions = tryFlatten(key, value, writer, depth, currentOptions, rootLiteralKeys, pathPrefix, + blockedKeys, remainingDepth, siblings); + if (currentOptions == null) { + return; + } + } + + dispatchValueEncoding(encodedKey, currentPath, key, value, writer, depth, currentOptions, + rootLiteralKeys, effectiveFlattenDepth, blockedKeys); + } + + /** + * Returns whether the flattening preconditions hold: a positive + * remaining depth, sibling collisions, unblocked key, non-null block + * set and a SAFE flatten strategy. + */ + private static boolean shouldTryFlatten(final String key, + final Set siblings, @Nullable final Set blockedKeys, + final int remainingDepth, final EncodeOptions options) { + return remainingDepth > 0 && !siblings.isEmpty() && blockedKeys != null && !blockedKeys.contains(key) - && KeyFolding.SAFE == currentOptions.flatten()) { - final Flatten.FoldResult foldResult = Flatten.tryFoldKeyChain(key, value, siblings, rootLiteralKeys, - pathPrefix, remainingDepth); - if (foldResult != null) { - currentOptions = flatten(key, foldResult, writer, depth, currentOptions, rootLiteralKeys, pathPrefix, - blockedKeys, remainingDepth); - if (currentOptions == null) { - return; - } - } + && KeyFolding.SAFE == options.flatten(); + } + + /** + * Attempts to fold the key chain and flatten the folded object. + * + * @return the changed EncodeOptions, null when encoding was completed + * by flattening, or the unchanged options when folding failed + */ + @Nullable + private static EncodeOptions tryFlatten(final String key, final JsonNode value, + final LineWriter writer, final int depth, final EncodeOptions options, + @Nullable final Set rootLiteralKeys, @Nullable final String pathPrefix, + @Nullable final Set blockedKeys, final int remainingDepth, final Set siblings) { + if (blockedKeys == null) { + return options; } + final Flatten.FoldResult foldResult = Flatten.tryFoldKeyChain(key, value, siblings, rootLiteralKeys, + pathPrefix, remainingDepth); + if (foldResult == null) { + return options; + } + return flatten(key, foldResult, writer, depth, options, rootLiteralKeys, pathPrefix, blockedKeys, + remainingDepth); + } + /** + * Dispatches the value encoding by node type: primitive, array or + * object (with keyed-tabular detection). + */ + private static void dispatchValueEncoding(final String encodedKey, final String currentPath, + final String key, final JsonNode value, final LineWriter writer, final int depth, + final EncodeOptions options, @Nullable final Set rootLiteralKeys, + final int effectiveFlattenDepth, @Nullable final Set blockedKeys) { if (value.isValueNode()) { writer.push(depth, encodedKey + COLON + SPACE - + PrimitiveEncoder.encodePrimitive(value, currentOptions.delimiter().toString())); + + PrimitiveEncoder.encodePrimitive(value, options.delimiter().toString())); } if (value.isArray()) { - ArrayEncoder.encodeArray(key, (ArrayNode) value, writer, depth, currentOptions); + ArrayEncoder.encodeArray(key, (ArrayNode) value, writer, depth, options); } if (value.isObject()) { final ObjectNode objValue = (ObjectNode) value; + final List keyedFields = KeyedObjectEncoder.detectKeyedFields(objValue); + if (!keyedFields.isEmpty()) { + KeyedObjectEncoder.encodeKeyedTabularObject(key, objValue, keyedFields, writer, depth, options); + return; + } writer.push(depth, encodedKey + COLON); if (!objValue.isEmpty()) { - encodeObject(objValue, writer, depth + 1, currentOptions, rootLiteralKeys, currentPath, + encodeObject(objValue, writer, depth + 1, options, rootLiteralKeys, currentPath, effectiveFlattenDepth, blockedKeys); } } diff --git a/src/main/java/dev/toonformat/jtoon/encoder/PrimitiveEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/PrimitiveEncoder.java index 26854695..3987b896 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/PrimitiveEncoder.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/PrimitiveEncoder.java @@ -154,7 +154,7 @@ public static String joinEncodedValues(final Collection values, final * * @param length Array length * @param key Optional key prefix - * @param fields Optional field names for tabular format + * @param fields Optional header fields for tabular format * @param delimiter The delimiter being used * @param lengthMarker Whether to include # marker before length * @return Formatted header string @@ -162,7 +162,7 @@ public static String joinEncodedValues(final Collection values, final public static String formatHeader( final int length, @Nullable final String key, - @Nullable final List fields, + @Nullable final List fields, final String delimiter, final boolean lengthMarker) { return HeaderFormatter.format(length, key, fields, delimiter, lengthMarker); diff --git a/src/main/java/dev/toonformat/jtoon/encoder/TabularArrayEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/TabularArrayEncoder.java index 22a04d7a..cf9001cf 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/TabularArrayEncoder.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/TabularArrayEncoder.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Detects and encodes uniform arrays of objects in efficient tabular format. @@ -22,11 +23,12 @@ private TabularArrayEncoder() { /** * Detects if an array can be encoded in tabular format. * Returns the header fields if tabular encoding is possible, empty list otherwise. + * Columns holding uniform nested objects are collapsed into nested field groups (§9.3). * * @param rows The array to analyze - * @return List of field names for tabular header, or empty list if not tabular + * @return List of header fields for tabular encoding, or empty list if not tabular */ - public static List detectTabularHeader(final ArrayNode rows) { + public static List detectTabularHeader(final ArrayNode rows) { if (rows.isEmpty()) { return Collections.emptyList(); } @@ -37,41 +39,71 @@ public static List detectTabularHeader(final ArrayNode rows) { } final ObjectNode firstObj = (ObjectNode) firstRow; - final List firstKeys = new ArrayList<>(firstObj.propertyNames()); - - if (firstKeys.isEmpty()) { + if (firstObj.isEmpty()) { return Collections.emptyList(); } - if (isTabularArray(rows, firstKeys)) { - return Collections.unmodifiableList(firstKeys); + final List header = new ArrayList<>(); + for (final String key : firstObj.propertyNames()) { + final Optional> children = uniformColumnsOf(firstObj.get(key)); + if (children.isEmpty()) { + return Collections.emptyList(); + } + header.add(new TabularField(key, children.get())); + } + + if (!matchesEveryRow(rows, header)) { + return Collections.emptyList(); } - return Collections.emptyList(); + return Collections.unmodifiableList(header); } /** - * Checks if all rows in the array have the same keys with primitive values. + * Derives the nested field-group structure of a single column value. + * Returns {@link Optional#empty()} when the column cannot be tabular (§9.3): + * arrays, empty objects, and values whose structure differs per row. + * A primitive (leaf) column yields an empty field list; a nested uniform + * object column yields its child fields. */ - private static boolean isTabularArray(final Iterable rows, final List header) { - final int headerSize = header.size(); + static Optional> uniformColumnsOf(final JsonNode value) { + if (value.isValueNode()) { + return Optional.of(Collections.emptyList()); + } + if (!value.isObject()) { + return Optional.empty(); + } + final ObjectNode obj = (ObjectNode) value; + if (obj.isEmpty()) { + return Optional.empty(); + } + final List children = new ArrayList<>(); + for (final String key : obj.propertyNames()) { + final Optional> subChildren = uniformColumnsOf(obj.get(key)); + if (subChildren.isEmpty()) { + return Optional.empty(); + } + children.add(new TabularField(key, subChildren.get())); + } + return Optional.of(children); + } - for (JsonNode row : rows) { + /** + * Checks that every row matches the header structure with uniform values. + */ + static boolean matchesEveryRow(final Iterable rows, final List header) { + for (final JsonNode row : rows) { if (!row.isObject()) { return false; } final ObjectNode obj = (ObjectNode) row; - - // All objects must have the same number of keys - if (obj.size() != headerSize) { + if (obj.size() != header.size()) { return false; } - // Check that all header keys exist in the row and all values are primitives - for (final String key : header) { - final JsonNode value = obj.get(key); - if (value == null || !value.isValueNode()) { + for (final TabularField field : header) { + if (!matchesField(field, obj.get(field.name()))) { return false; } } @@ -80,18 +112,44 @@ private static boolean isTabularArray(final Iterable rows, final List< return true; } + /** + * Checks that a row value matches the field structure: primitives for leaf + * fields, uniformly structured objects for nested field groups. + */ + private static boolean matchesField(final TabularField field, @Nullable final JsonNode value) { + if (value == null) { + return false; + } + if (field.isLeaf()) { + return value.isValueNode(); + } + if (!value.isObject()) { + return false; + } + final ObjectNode obj = (ObjectNode) value; + if (obj.size() != field.children().size()) { + return false; + } + for (final TabularField child : field.children()) { + if (!matchesField(child, obj.get(child.name()))) { + return false; + } + } + return true; + } + /** * Encodes an array of objects as a tabular structure. * * @param prefix Optional key prefix * @param rows Array of uniform objects - * @param header List of field names + * @param header List of header fields * @param writer LineWriter for output * @param depth Indentation depth * @param options Encoding options */ public static void encodeArrayOfObjectsAsTabular(@Nullable final String prefix, final ArrayNode rows, - final List header, final LineWriter writer, final int depth, + final List header, final LineWriter writer, final int depth, final EncodeOptions options) { final String headerStr = PrimitiveEncoder.formatHeader(rows.size(), prefix, header, options.delimiter().toString(), options.lengthMarker()); @@ -101,16 +159,16 @@ public static void encodeArrayOfObjectsAsTabular(@Nullable final String prefix, } /** - * Writes rows of tabular data by extracting values in header order. + * Writes rows of tabular data by extracting leaf values in header order. * Public to allow ListItemEncoder to write rows after placing header on "- " line. * * @param rows Array of objects - * @param header List of field names + * @param header List of header fields * @param writer LineWriter for output * @param depth Indentation depth * @param options Encoding options */ - public static void writeTabularRows(final Iterable rows, final List header, + public static void writeTabularRows(final Iterable rows, final List header, final LineWriter writer, final int depth, final EncodeOptions options) { for (JsonNode row : rows) { // Skip non-object rows @@ -124,25 +182,31 @@ public static void writeTabularRows(final Iterable rows, final List header, final String delimiter) { - final StringBuilder sb = new StringBuilder(128); - boolean first = true; - for (final String key : header) { - final JsonNode value = row.get(key); - if (value == null) { - continue; // Skip missing keys + private static String joinRowValues(final ObjectNode row, final List header, final String delimiter) { + final List cells = new ArrayList<>(header.size()); + collectCells(row, header, cells, delimiter); + return String.join(delimiter, cells); + } + + static void collectCells(final ObjectNode row, final List fields, final List cells, + final String delimiter) { + for (final TabularField field : fields) { + if (field.isLeaf()) { + final JsonNode value = row.get(field.name()); + if (value != null) { + cells.add(PrimitiveEncoder.encodePrimitive(value, delimiter)); + } + continue; } - if (!first) { - sb.append(delimiter); + final JsonNode group = row.get(field.name()); + if (group != null && group.isObject()) { + collectCells((ObjectNode) group, field.children(), cells, delimiter); } - first = false; - sb.append(PrimitiveEncoder.encodePrimitive(value, delimiter)); } - return sb.toString(); } } diff --git a/src/main/java/dev/toonformat/jtoon/encoder/TabularField.java b/src/main/java/dev/toonformat/jtoon/encoder/TabularField.java new file mode 100644 index 00000000..d4194a74 --- /dev/null +++ b/src/main/java/dev/toonformat/jtoon/encoder/TabularField.java @@ -0,0 +1,33 @@ +package dev.toonformat.jtoon.encoder; + +import java.util.List; + +/** + * A field of a tabular header. Leaf fields hold primitive values encoded as a single + * cell; non-leaf fields represent nested uniform object columns collapsed into nested + * field groups (§9.3 of the TOON specification). + * + * @param name Field name + * @param children Child fields of the nested group; empty for a leaf field + */ +public record TabularField(String name, List children) { + + /** + * Creates a leaf field whose values are encoded as a single tabular cell. + * + * @param name Field name + * @return Leaf field with no children + */ + public static TabularField leaf(final String name) { + return new TabularField(name, List.of()); + } + + /** + * Checks whether this field is a leaf (single-cell primitive column). + * + * @return true if the field has no child fields + */ + public boolean isLeaf() { + return children.isEmpty(); + } +} diff --git a/src/main/java/dev/toonformat/jtoon/encoder/ValueEncoder.java b/src/main/java/dev/toonformat/jtoon/encoder/ValueEncoder.java index 2b558f48..e89148e3 100644 --- a/src/main/java/dev/toonformat/jtoon/encoder/ValueEncoder.java +++ b/src/main/java/dev/toonformat/jtoon/encoder/ValueEncoder.java @@ -5,6 +5,7 @@ import tools.jackson.databind.node.ArrayNode; import tools.jackson.databind.node.ObjectNode; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -41,8 +42,14 @@ public static String encodeValue(final JsonNode value, final EncodeOptions optio if (value.isArray()) { ArrayEncoder.encodeArray(null, (ArrayNode) value, writer, 0, options); } else if (value.isObject()) { - final Set jsonNodes = new HashSet<>(value.propertyNames()); - ObjectEncoder.encodeObject((ObjectNode) value, writer, 0, options, jsonNodes, null, null, new HashSet<>()); + final ObjectNode obj = (ObjectNode) value; + final List keyedFields = KeyedObjectEncoder.detectKeyedFields(obj); + if (!keyedFields.isEmpty()) { + KeyedObjectEncoder.encodeKeyedTabularObject(null, obj, keyedFields, writer, 0, options); + } else { + final Set jsonNodes = new HashSet<>(value.propertyNames()); + ObjectEncoder.encodeObject(obj, writer, 0, options, jsonNodes, null, null, new HashSet<>()); + } } return writer.toString(); diff --git a/src/main/java/dev/toonformat/jtoon/normalizer/JsonNormalizer.java b/src/main/java/dev/toonformat/jtoon/normalizer/JsonNormalizer.java index d35892a4..4411cede 100644 --- a/src/main/java/dev/toonformat/jtoon/normalizer/JsonNormalizer.java +++ b/src/main/java/dev/toonformat/jtoon/normalizer/JsonNormalizer.java @@ -339,61 +339,104 @@ private static JsonNode tryNormalizePojo(final Object value) { */ private static JsonNode normalizeArray(final Object array) { if (array instanceof int[] intArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (int i : intArray) { - node.add(IntNode.valueOf(i)); - } - return node; - } else if (array instanceof long[] longArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (long l : longArray) { - node.add(LongNode.valueOf(l)); - } - return node; - } else if (array instanceof double[] doubleArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (final double d : doubleArray) { - node.add(Double.isFinite(d) ? DoubleNode.valueOf(d) : NullNode.getInstance()); - } - return node; - } else if (array instanceof float[] floatArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (final float f : floatArray) { - node.add(Float.isFinite(f) ? FloatNode.valueOf(f) : NullNode.getInstance()); - } - return node; - } else if (array instanceof boolean[] boolArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (boolean b : boolArray) { - node.add(BooleanNode.valueOf(b)); - } - return node; - } else if (array instanceof byte[] byteArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (byte by : byteArray) { - node.add(IntNode.valueOf(by)); - } - return node; - } else if (array instanceof short[] shortArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (short s : shortArray) { - node.add(ShortNode.valueOf(s)); - } - return node; - } else if (array instanceof char[] charArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (char c : charArray) { - node.add(StringNode.valueOf(String.valueOf(c))); - } - return node; - } else if (array instanceof Object[] objArray) { - final ArrayNode node = MAPPER.createArrayNode(); - for (Object o : objArray) { - node.add(normalize(o)); - } - return node; - } else { - return MAPPER.createArrayNode(); + return intArrayNode(intArray); + } + if (array instanceof long[] longArray) { + return longArrayNode(longArray); + } + if (array instanceof double[] doubleArray) { + return doubleArrayNode(doubleArray); + } + if (array instanceof float[] floatArray) { + return floatArrayNode(floatArray); + } + if (array instanceof boolean[] boolArray) { + return boolArrayNode(boolArray); + } + if (array instanceof byte[] byteArray) { + return byteArrayNode(byteArray); + } + if (array instanceof short[] shortArray) { + return shortArrayNode(shortArray); + } + if (array instanceof char[] charArray) { + return charArrayNode(charArray); + } + if (array instanceof Object[] objArray) { + return objectArrayNode(objArray); + } + return MAPPER.createArrayNode(); + } + + private static JsonNode intArrayNode(final int... intArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (int i : intArray) { + node.add(IntNode.valueOf(i)); + } + return node; + } + + private static JsonNode longArrayNode(final long... longArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (long l : longArray) { + node.add(LongNode.valueOf(l)); + } + return node; + } + + private static JsonNode doubleArrayNode(final double... doubleArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (final double d : doubleArray) { + node.add(Double.isFinite(d) ? DoubleNode.valueOf(d) : NullNode.getInstance()); + } + return node; + } + + private static JsonNode floatArrayNode(final float... floatArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (final float f : floatArray) { + node.add(Float.isFinite(f) ? FloatNode.valueOf(f) : NullNode.getInstance()); + } + return node; + } + + private static JsonNode boolArrayNode(final boolean... boolArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (boolean b : boolArray) { + node.add(BooleanNode.valueOf(b)); + } + return node; + } + + private static JsonNode byteArrayNode(final byte... byteArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (byte by : byteArray) { + node.add(IntNode.valueOf(by)); + } + return node; + } + + private static JsonNode shortArrayNode(final short... shortArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (short s : shortArray) { + node.add(ShortNode.valueOf(s)); + } + return node; + } + + private static JsonNode charArrayNode(final char... charArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (char c : charArray) { + node.add(StringNode.valueOf(String.valueOf(c))); + } + return node; + } + + private static JsonNode objectArrayNode(final Object... objArray) { + final ArrayNode node = MAPPER.createArrayNode(); + for (Object o : objArray) { + node.add(normalize(o)); } + return node; } } diff --git a/src/main/java/dev/toonformat/jtoon/util/Headers.java b/src/main/java/dev/toonformat/jtoon/util/Headers.java index 0e556585..324e98fb 100644 --- a/src/main/java/dev/toonformat/jtoon/util/Headers.java +++ b/src/main/java/dev/toonformat/jtoon/util/Headers.java @@ -1,6 +1,7 @@ package dev.toonformat.jtoon.util; import java.util.regex.Pattern; +import org.jspecify.annotations.Nullable; /** * Patterns in form of regex that must be followed in order to decode arrays, tabular, keyed arrays. @@ -21,14 +22,299 @@ public final class Headers { public static final Pattern TABULAR_HEADER_PATTERN = Pattern.compile("^\\[(#?)(\\d+)([\\t|])?]\\{(.+)}:"); /** - * Matches keyed array headers: items[2]{id,name}: or tags[3]: or data[4]{id}:. - * Also matches quoted keys with brackets: "key[test]"[3]: and keys with - * escaped quotes: "key\"quote"[3]:. - * Captures: group(1)=key (quoted or unquoted), group(2)=#marker, group(3)=delimiter, - * group(4)=optional field spec + * The keyed header scanner {@link #matchKeyedArrayHeader} replaces the + * regex-based dispatch: a field spec may nest braces at any depth + * ({@code {geo{point{lat,lon}}}}, §6, §9.3), which a flat regex cannot + * express. The former KEYED_ARRAY_PATTERN is kept only for its key and + * bracket-segment grammar, documented below. + * Matches keyed array headers: items[2]{id,name}: or tags[3]:. + * Group 1: key, Group 2: #marker, Group 3: delimiter, Group 4: flat field spec. */ public static final Pattern KEYED_ARRAY_PATTERN = Pattern.compile( - "^(\"(?:[^\"\\\\]|\\\\.)*+\"|[^\\[\\]]++)\\[(#?)\\d++([\\t|])?](\\{[^}]+})?+:.*+$"); + "^(\"(?:[^\"\\\\]|\\\\.)*+\"|[^\\[\\]:\\s]++)\\[(#?)\\d++([\\t|])?](\\{[^}]+})?:.*+$"); + + /** + * Result of {@link #matchKeyedArrayHeader} and {@link #matchKeylessKeyedHeader}: + * the matched key, the structural positions of the header segments, and the + * declarations extracted from the bracket segment. + * + * @param key the matched key, quoted or unquoted; empty for a keyless header + * @param keyEnd the index just past the key, where the bracket segment starts + * @param declaredLength the declared entry/row count inside the bracket segment + * @param keyed whether the bracket segment declares a keyed marker (§9.5) + * @param delimiter the delimiter declared inside the bracket segment, or null for the default + * @param fieldsStart the index of the field-spec opening brace, or -1 without a field spec + * @param headerEnd the index just past the field spec, or just past the bracket segment + */ + public record KeyedHeaderMatch(String key, int keyEnd, long declaredLength, boolean keyed, + @Nullable Character delimiter, int fieldsStart, int headerEnd) { + } + + /** + * Scans a line for a keyed header ({@code key[N]{field…}: …}) + * per the §6 grammar, accepting field specs with braces nested at any + * depth. Braces, delimiters and colons inside quoted field names do not + * count as structure. The bracket segment may declare a keyed marker + * ({@code [N:]} or {@code [N:delim]}, §9.5). + * + * @param content the line content to scan + * @return the keyed header match, or null when the content is not a keyed header + */ + @Nullable + public static KeyedHeaderMatch matchKeyedArrayHeader(final String content) { + return scanHeader(content, true); + } + + /** + * Scans a keyless header that starts with a bracket segment, for + * root-form discovery (§5). The match is returned whether or not it + * declares a keyed marker; callers must test {@code keyed()} to tell + * {@code [2:]{…}:} (keyed tabular object, §9.5) from {@code [2]{…}:} + * (plain keyless tabular array). + * + * @param content the line content to scan + * @return the keyless header match, or null when the content is not a keyless header + */ + @Nullable + public static KeyedHeaderMatch matchKeylessKeyedHeader(final String content) { + return scanHeader(content, false); + } + + @Nullable + private static KeyedHeaderMatch scanHeader(final String content, final boolean requireKey) { + final int n = content.length(); + final int keyEnd = scanKey(content, n, requireKey); + if (keyEnd < 0) { + return null; + } + + final BracketSegment bracket = scanBracketSegment(content, keyEnd, n); + if (bracket == null) { + return null; + } + + final FieldSpecMatch fields = scanFieldSpec(content, bracket.endIndex(), n); + if (fields == null) { + return null; + } + + // Trailing colon required after the bracket or field spec segment + if (fields.endIndex() >= n || content.charAt(fields.endIndex()) != ':') { + return null; + } + return new KeyedHeaderMatch(content.substring(0, keyEnd), keyEnd, bracket.declaredLength(), + bracket.keyed(), bracket.delimiter(), fields.start(), fields.endIndex()); + } + + /** + * Scans the key segment: a quoted key with escapes honored (§7.3), an + * unquoted {@code [^\[\]:\s]+} key, or no key at all for keyless headers. + * + * @param content the line content to scan + * @param n the content length + * @param requireKey whether a key must be present + * @return the index just past the key, or -1 when the key is missing or + * a quoted key is unterminated + */ + private static int scanKey(final String content, final int n, final boolean requireKey) { + if (!requireKey) { + return 0; + } + if (content.charAt(0) == '"') { + return scanQuotedKey(content, n); + } + return scanUnquotedKey(content, 0, n); + } + + /** + * Scans a quoted key up to its unescaped closing quote. + * + * @param content the line content to scan + * @param n the content length + * @return the index just past the closing quote, or -1 when unterminated + */ + private static int scanQuotedKey(final String content, final int n) { + int i = 1; + boolean escaped = false; + while (i < n) { + final char c = content.charAt(i); + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '"') { + return i + 1; + } + i++; + } + return -1; + } + + /** + * Scans an unquoted key up to a structural character or whitespace. + * + * @param content the line content to scan + * @param keyStart the index where the key starts + * @param n the content length + * @return the index just past the key, or -1 when the key is empty + */ + private static int scanUnquotedKey(final String content, final int keyStart, final int n) { + int i = keyStart; + while (i < n && content.charAt(i) != '[' && content.charAt(i) != ':' + && !Character.isWhitespace(content.charAt(i))) { + i++; + } + if (i == keyStart) { + return -1; + } + return i; + } + + /** + * Scans the bracket segment: {@code [ (#?) \d+ (:)? ([\t|])? ]}. + * + * @param content the line content to scan + * @param start the index of the opening bracket + * @param n the content length + * @return the parsed segment, or null for a malformed segment + */ + @Nullable + private static BracketSegment scanBracketSegment(final String content, final int start, final int n) { + int i = start; + if (i >= n || content.charAt(i) != '[') { + return null; + } + i = skipHashMarker(content, i + 1, n); + final int digitsStart = i; + while (i < n && Character.isDigit(content.charAt(i))) { + i++; + } + if (i == digitsStart) { + return null; + } + final long declaredLength; + try { + declaredLength = Long.parseLong(content.substring(digitsStart, i)); + } catch (NumberFormatException e) { + return null; + } + boolean keyed = false; + if (i < n && content.charAt(i) == ':') { + keyed = true; + i++; + } + @Nullable Character delimiter = null; + if (i < n && (content.charAt(i) == '\t' || content.charAt(i) == '|')) { + delimiter = content.charAt(i); + i++; + } + if (i >= n || content.charAt(i) != ']') { + return null; + } + return new BracketSegment(declaredLength, keyed, delimiter, i + 1); + } + + /** + * Skips an optional length-marker hash in the bracket segment. + * + * @param content the line content to scan + * @param i the index to inspect + * @param n the content length + * @return the index just past the hash, or the unchanged index + */ + private static int skipHashMarker(final String content, final int i, final int n) { + if (i < n && content.charAt(i) == '#') { + return i + 1; + } + return i; + } + + /** + * Scans the optional balanced field spec {@code {…}} with braces nested + * at any depth; at least one field entry is required (§6). + * + * @param content the line content to scan + * @param start the index where the field spec may start + * @param n the content length + * @return the parsed segment, or null for a malformed field spec + */ + @Nullable + private static FieldSpecMatch scanFieldSpec(final String content, final int start, final int n) { + if (start >= n || content.charAt(start) != '{') { + return new FieldSpecMatch(-1, start); + } + final int closingBrace = skipBalancedFieldSpec(content, start + 1, n); + if (closingBrace < 0 || closingBrace - start <= 1) { + return null; + } + return new FieldSpecMatch(start, closingBrace + 1); + } + + /** + * Skips a balanced brace group, honoring quoted field names and escaped + * characters, and returns the index of its closing brace. + * + * @param content the line content to scan + * @param i the index just past the opening brace + * @param n the content length + * @return the index of the matching closing brace, or -1 when unbalanced + */ + private static int skipBalancedFieldSpec(final String content, final int i, final int n) { + int pos = i; + int depth = 1; + boolean escaped = false; + boolean inQuotes = false; + while (pos < n) { + final char c = content.charAt(pos); + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '"') { + inQuotes = !inQuotes; + } else if (!inQuotes && (c == '{' || c == '}')) { + depth = adjustDepth(depth, c); + if (depth == 0) { + return pos; + } + } + pos++; + } + return -1; + } + + /** + * Adjusts the brace depth for an opening or closing brace. + * + * @param depth the current brace depth + * @param c the brace character + * @return the adjusted depth + */ + private static int adjustDepth(final int depth, final char c) { + return c == '}' ? depth - 1 : depth + 1; + } + + /** + * The parsed bracket segment of a keyed header. + * + * @param declaredLength the declared entry/row count + * @param keyed whether the segment declares a keyed marker (§9.5) + * @param delimiter the declared delimiter, or null for the default + * @param endIndex the index just past the closing bracket + */ + private record BracketSegment(long declaredLength, boolean keyed, + @Nullable Character delimiter, int endIndex) { + } + + /** + * The parsed field spec segment of a keyed header. + * + * @param start the index of the opening brace, or -1 without a field spec + * @param endIndex the index just past the closing brace, or just past the + * preceding segment without a field spec + */ + private record FieldSpecMatch(int start, int endIndex) { + } private Headers() { throw new UnsupportedOperationException("Utility class cannot be instantiated"); diff --git a/src/main/java/dev/toonformat/jtoon/util/StringEscaper.java b/src/main/java/dev/toonformat/jtoon/util/StringEscaper.java index d3f00f4d..2a424d61 100644 --- a/src/main/java/dev/toonformat/jtoon/util/StringEscaper.java +++ b/src/main/java/dev/toonformat/jtoon/util/StringEscaper.java @@ -63,67 +63,107 @@ public static void validateString(final String value) { if (value == null || value.isEmpty()) { return; } + validateQuotedString(value); + } - // Check for unterminated string (starts with quote but doesn't end with quote) - if (value.startsWith("\"") && !value.endsWith("\"")) { - throw new IllegalArgumentException("Unterminated string"); + /** + * Validates a quoted string for untermination and invalid escape + * sequences; unquoted values pass through. + * + * @param value the string to validate + */ + private static void validateQuotedString(final String value) { + if (!value.startsWith("\"") || !value.endsWith("\"")) { + // Check for unterminated string (starts with quote but doesn't end with quote) + if (value.startsWith("\"")) { + throw new IllegalArgumentException("Unterminated string"); + } + return; } + scanForInvalidEscapes(value.substring(1, value.length() - 1)); + } - // Check for invalid escape sequences in quoted strings - if (value.startsWith("\"") && value.endsWith("\"")) { - final String unquoted = value.substring(1, value.length() - 1); - boolean escaped = false; - int i = 0; - while (i < unquoted.length()) { - final char c = unquoted.charAt(i); - if (escaped) { - // Check if escape sequence is valid - if (!isValidEscapeChar(c)) { - throw new IllegalArgumentException("Invalid escape sequence: \\" + c); - } - if (c == 'u') { - if (i + UNICODE_HEX_LENGTH >= unquoted.length()) { - throw new IllegalArgumentException(INVALID_ESCAPE_U); - } - final String hex = unquoted.substring(i + 1, i + 1 + UNICODE_HEX_LENGTH); - if (!isHexString(hex)) { - throw new IllegalArgumentException(INVALID_ESCAPE_U + hex); - } - final int codePoint = Integer.parseInt(hex, HEX_RADIX); - if (Character.isLowSurrogate((char) codePoint)) { - throw new IllegalArgumentException(INVALID_UNICODE_LONE_LOW); - } - if (Character.isHighSurrogate((char) codePoint)) { - final int nextEscapeStart = i + 1 + UNICODE_HEX_LENGTH; - if (nextEscapeStart + UNICODE_ESCAPE_TOTAL_LENGTH - 1 >= unquoted.length() - || unquoted.charAt(nextEscapeStart) != '\\' - || unquoted.charAt(nextEscapeStart + 1) != 'u') { - throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); - } - final String nextHex = unquoted.substring(nextEscapeStart + 2, - nextEscapeStart + 2 + UNICODE_HEX_LENGTH); - if (!isHexString(nextHex) - || !Character.isLowSurrogate((char) Integer.parseInt(nextHex, HEX_RADIX))) { - throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); - } - // Skip past the full surrogate pair (\\uXXXX\\uXXXX = 12 chars total) - // to avoid reprocessing the consumed hex digits and the low surrogate - // escape as individual characters. - i += UNICODE_ESCAPE_TOTAL_LENGTH + UNICODE_HEX_LENGTH; - } - } - escaped = false; - } else if (c == '\\') { - escaped = true; + /** + * Scans an unquoted string content for invalid escape sequences, + * including a trailing backslash. + * + * @param unquoted the unquoted string content + */ + private static void scanForInvalidEscapes(final String unquoted) { + boolean escaped = false; + int i = 0; + while (i < unquoted.length()) { + final char c = unquoted.charAt(i); + if (escaped) { + // Check if escape sequence is valid + validateEscapeSequence(c); + if (c == 'u') { + i = validateUnicodeEscape(unquoted, i); } - i++; + escaped = false; + } else if (c == '\\') { + escaped = true; } + i++; + } - // Check for trailing backslash (invalid escape) - if (escaped) { - throw new IllegalArgumentException("Invalid escape sequence: trailing backslash"); + // Check for trailing backslash (invalid escape) + if (escaped) { + throw new IllegalArgumentException("Invalid escape sequence: trailing backslash"); + } + } + + /** + * Rejects characters that are not valid after a backslash. + * + * @param c the character following a backslash + */ + private static void validateEscapeSequence(final char c) { + if (!isValidEscapeChar(c)) { + throw new IllegalArgumentException("Invalid escape sequence: \\" + c); + } + } + + /** + * Validates the {@code \\uXXXX} escape starting at the given index, + * including a following low-surrogate escape of a surrogate pair. + * + * @param unquoted the unquoted string content + * @param i the index of the 'u' of the escape + * @return the index to continue scanning from, after a validated + * surrogate pair; unchanged for a single code unit + */ + private static int validateUnicodeEscape(final String unquoted, final int i) { + if (i + UNICODE_HEX_LENGTH >= unquoted.length()) { + throw new IllegalArgumentException(INVALID_ESCAPE_U); + } + final String hex = unquoted.substring(i + 1, i + 1 + UNICODE_HEX_LENGTH); + if (!isHexString(hex)) { + throw new IllegalArgumentException(INVALID_ESCAPE_U + hex); + } + final int codePoint = Integer.parseInt(hex, HEX_RADIX); + if (Character.isLowSurrogate((char) codePoint)) { + throw new IllegalArgumentException(INVALID_UNICODE_LONE_LOW); + } + if (Character.isHighSurrogate((char) codePoint)) { + final int nextEscapeStart = i + 1 + UNICODE_HEX_LENGTH; + if (nextEscapeStart + UNICODE_ESCAPE_TOTAL_LENGTH - 1 >= unquoted.length() + || unquoted.charAt(nextEscapeStart) != '\\' + || unquoted.charAt(nextEscapeStart + 1) != 'u') { + throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); + } + final String nextHex = unquoted.substring(nextEscapeStart + 2, + nextEscapeStart + 2 + UNICODE_HEX_LENGTH); + if (!isHexString(nextHex) + || !Character.isLowSurrogate((char) Integer.parseInt(nextHex, HEX_RADIX))) { + throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); } + // Skip past the full surrogate pair (\\uXXXX\\uXXXX = 12 chars total) + // to avoid reprocessing the consumed hex digits and the low surrogate + // escape as individual characters. + return i + UNICODE_ESCAPE_TOTAL_LENGTH + UNICODE_HEX_LENGTH; } + return i; } /** @@ -158,39 +198,7 @@ public static String unescape(final String value) { final char c = unquoted.charAt(i); if (escaped) { if (c == 'u') { - if (i + UNICODE_HEX_LENGTH >= unquoted.length()) { - throw new IllegalArgumentException(INVALID_ESCAPE_U); - } - final String hex = unquoted.substring(i + 1, i + 1 + UNICODE_HEX_LENGTH); - if (!isHexString(hex)) { - throw new IllegalArgumentException(INVALID_ESCAPE_U + hex); - } - final char codeUnit = (char) Integer.parseInt(hex, HEX_RADIX); - if (Character.isLowSurrogate(codeUnit)) { - throw new IllegalArgumentException(INVALID_UNICODE_LONE_LOW); - } - if (Character.isHighSurrogate(codeUnit)) { - if (i + (2 * UNICODE_ESCAPE_TOTAL_LENGTH) - 2 >= unquoted.length() - || unquoted.charAt(i + 1 + UNICODE_HEX_LENGTH) != '\\' - || unquoted.charAt(i + 2 + UNICODE_HEX_LENGTH) != 'u') { - throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); - } - final String lowHex = unquoted.substring(i + 3 + UNICODE_HEX_LENGTH, - i + 3 + (2 * UNICODE_HEX_LENGTH)); - if (!isHexString(lowHex)) { - throw new IllegalArgumentException(INVALID_ESCAPE_U + lowHex); - } - final char lowCodeUnit = (char) Integer.parseInt(lowHex, HEX_RADIX); - if (!Character.isLowSurrogate(lowCodeUnit)) { - throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); - } - result.append(codeUnit); - result.append(lowCodeUnit); - i += (2 * UNICODE_ESCAPE_TOTAL_LENGTH) - 2; - } else { - result.append(codeUnit); - i += UNICODE_HEX_LENGTH; - } + i = appendUnicodeEscape(result, unquoted, i); } else { result.append(unescapeChar(c)); } @@ -206,6 +214,51 @@ public static String unescape(final String value) { return result.toString(); } + /** + * Appends the decoded {@code \\uXXXX} escape starting at the given index, + * including a following low-surrogate escape of a surrogate pair. + * + * @param result the builder receiving the decoded characters + * @param unquoted the unquoted string content + * @param i the index of the 'u' of the escape + * @return the index to continue scanning from, after a decoded surrogate + * pair; otherwise the index of the last hex digit + */ + private static int appendUnicodeEscape(final StringBuilder result, final String unquoted, final int i) { + if (i + UNICODE_HEX_LENGTH >= unquoted.length()) { + throw new IllegalArgumentException(INVALID_ESCAPE_U); + } + final String hex = unquoted.substring(i + 1, i + 1 + UNICODE_HEX_LENGTH); + if (!isHexString(hex)) { + throw new IllegalArgumentException(INVALID_ESCAPE_U + hex); + } + final char codeUnit = (char) Integer.parseInt(hex, HEX_RADIX); + if (Character.isLowSurrogate(codeUnit)) { + throw new IllegalArgumentException(INVALID_UNICODE_LONE_LOW); + } + if (Character.isHighSurrogate(codeUnit)) { + if (i + (2 * UNICODE_ESCAPE_TOTAL_LENGTH) - 2 >= unquoted.length() + || unquoted.charAt(i + 1 + UNICODE_HEX_LENGTH) != '\\' + || unquoted.charAt(i + 2 + UNICODE_HEX_LENGTH) != 'u') { + throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); + } + final String lowHex = unquoted.substring(i + 3 + UNICODE_HEX_LENGTH, + i + 3 + (2 * UNICODE_HEX_LENGTH)); + if (!isHexString(lowHex)) { + throw new IllegalArgumentException(INVALID_ESCAPE_U + lowHex); + } + final char lowCodeUnit = (char) Integer.parseInt(lowHex, HEX_RADIX); + if (!Character.isLowSurrogate(lowCodeUnit)) { + throw new IllegalArgumentException(INVALID_UNICODE_LONE_HIGH); + } + result.append(codeUnit); + result.append(lowCodeUnit); + return i + (2 * UNICODE_ESCAPE_TOTAL_LENGTH) - 2; + } + result.append(codeUnit); + return i + UNICODE_HEX_LENGTH; + } + private static boolean isHexString(final String value) { for (int i = 0; i < value.length(); i++) { if (Character.digit(value.charAt(i), HEX_RADIX) == -1) { diff --git a/src/main/java/dev/toonformat/jtoon/util/StringValidator.java b/src/main/java/dev/toonformat/jtoon/util/StringValidator.java index 7c7e61b8..76ecff37 100644 --- a/src/main/java/dev/toonformat/jtoon/util/StringValidator.java +++ b/src/main/java/dev/toonformat/jtoon/util/StringValidator.java @@ -45,6 +45,28 @@ public static boolean isSafeUnquoted(final String value, final String delimiter) return false; } + // Spec §7.2: tokens starting with '#' must be quoted (comment marker). + if (value.charAt(0) == '#') { + return false; + } + + if (!hasSafeCharacters(value, delimiter)) { + return false; + } + + return !value.startsWith(LIST_ITEM_MARKER); + } + + /** + * Rejects any character that would require quoting: structural + * characters, control characters and the active delimiter. + * + * @param value the string value to check + * @param delimiter the delimiter being used (for validation) + * @return true when every character is safe unquoted + */ + private static boolean hasSafeCharacters(final String value, final String delimiter) { + final int len = value.length(); for (int i = 0; i < len; i++) { final char c = value.charAt(i); switch (c) { @@ -61,8 +83,7 @@ public static boolean isSafeUnquoted(final String value, final String delimiter) } } } - - return !value.startsWith(LIST_ITEM_MARKER); + return true; } /** @@ -79,19 +100,14 @@ public static boolean isValidUnquotedKey(final String key) { final int len = key.length(); final char first = key.charAt(0); - if (!Character.isJavaIdentifierStart(first) && first != '_') { + // Spec §7.3: unquoted keys must match ^[A-Za-z_][A-Za-z0-9_.]*$ (ASCII only). + if (!isAsciiLetter(first) && first != '_') { return false; } for (int i = 1; i < len; i++) { final char c = key.charAt(i); - // Reject control characters (U+0000-U+001F) even though - // Character.isJavaIdentifierPart returns true for identifier-ignorable - // control chars like U+0004. These must be escaped in TOON output. - if (c <= CONTROL_CHAR_MAX) { - return false; - } - if (!Character.isJavaIdentifierPart(c) && c != '.') { + if (!isAsciiLetterOrDigit(c) && c != '_' && c != '.') { return false; } } @@ -99,6 +115,14 @@ public static boolean isValidUnquotedKey(final String key) { return true; } + private static boolean isAsciiLetter(final char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + private static boolean isAsciiLetterOrDigit(final char c) { + return isAsciiLetter(c) || (c >= '0' && c <= '9'); + } + private static boolean isKeyword(final String value) { return TRUE_LITERAL.equals(value) || FALSE_LITERAL.equals(value) @@ -109,51 +133,110 @@ private static boolean isNumericLike(final String value) { if (value.isEmpty()) { return false; } + final int start = skipSign(value); + if (start < 0) { + return false; + } + return scanNumericBody(value, start); + } - final int len = value.length(); - int i = 0; + /** + * Skips an optional leading sign. A lone sign is not numeric-like. + * + * @param value the value to inspect + * @return the index after the sign, 0 without a sign, or -1 for a lone sign + */ + private static int skipSign(final String value) { + final char first = value.charAt(0); + if (first != '-' && first != '+') { + return 0; + } + if (value.length() < 2) { + return -1; + } + return 1; + } - if (value.charAt(0) == '-') { - if (len < 2) { + /** + * Scans the numeric body after the optional sign. Mirrors the grammar + * [0-9]+ ('.' [0-9]+)? ([eE] [+-]? [0-9]+)? from Spec §7.2. + * + * @param value the value to scan + * @param start the index where the body starts + * @return true when the body is numeric-like + */ + private static boolean scanNumericBody(final String value, final int start) { + int i = consumeDigits(value, start); + if (i == start) { + return false; + } + if (i < value.length() && value.charAt(i) == '.') { + final int afterDot = consumeDigits(value, i + 1); + if (afterDot == i + 1) { return false; } - i = 1; + i = afterDot; } - - boolean hasDigit = false; - boolean hasDot = false; - boolean hasExponent = false; - - while (i < len) { - final char c = value.charAt(i); - - if (c >= '0' && c <= '9') { - hasDigit = true; - } else if (c == '.') { - if (hasDot || hasExponent || !hasDigit) { - return false; - } - hasDot = true; - hasDigit = false; - } else if (c == 'e' || c == 'E') { - if (!hasDigit || hasExponent) { - return false; - } - hasExponent = true; - hasDigit = false; - if (i + 1 < len) { - final char next = value.charAt(i + 1); - if (next == '+' || next == '-') { - i++; - } - } - } else { + if (i < value.length() && isExponentMarker(value.charAt(i))) { + final int afterExponent = consumeExponentPart(value, i + 1); + if (afterExponent == i + 1) { return false; } + i = afterExponent; + } + return i == value.length(); + } + + /** + * Consumes a run of ASCII digits. + * + * @param value the value to scan + * @param from the index to start at + * @return the index after the last consumed digit + */ + private static int consumeDigits(final String value, final int from) { + int i = from; + while (i < value.length() && isDigit(value.charAt(i))) { i++; } + return i; + } + + private static boolean isDigit(final char c) { + return c >= '0' && c <= '9'; + } - return hasDigit; + private static boolean isExponentMarker(final char c) { + return c == 'e' || c == 'E'; + } + + /** + * Consumes the exponent part after the marker: an optional sign + * followed by at least one digit. + * + * @param value the value to scan + * @param from the index after the exponent marker + * @return the index after the last consumed digit + */ + private static int consumeExponentPart(final String value, final int from) { + return consumeDigits(value, skipOptionalSign(value, from)); + } + + /** + * Skips an optional exponent sign. + * + * @param value the value to scan + * @param from the index to start at + * @return the index after the sign, or the unchanged index + */ + private static int skipOptionalSign(final String value, final int from) { + if (from < value.length()) { + final char c = value.charAt(from); + if (c == '+' || c == '-') { + return from + 1; + } + } + return from; } static boolean containsQuotesOrBackslash(final String value) { diff --git a/src/test/java/dev/toonformat/jtoon/JToonTest.java b/src/test/java/dev/toonformat/jtoon/JToonTest.java index d7a421e4..328d973e 100644 --- a/src/test/java/dev/toonformat/jtoon/JToonTest.java +++ b/src/test/java/dev/toonformat/jtoon/JToonTest.java @@ -507,7 +507,7 @@ void usesListForDifferentFields() { } @Test - @DisplayName("uses list format for objects with nested values") + @DisplayName("uses tabular format for uniform objects with nested values") void usesListForNestedValues() { // Given final Map obj = obj( @@ -517,10 +517,8 @@ void usesListForNestedValues() { // Then assertEquals( """ - items[1]: - - id: 1 - nested: - x: 1""", + items[1]{id,nested{x}}: + 1,1""", encode(obj)); } diff --git a/src/test/java/dev/toonformat/jtoon/conformance/ConformanceTest.java b/src/test/java/dev/toonformat/jtoon/conformance/ConformanceTest.java index c4cb8e94..6619c1f5 100644 --- a/src/test/java/dev/toonformat/jtoon/conformance/ConformanceTest.java +++ b/src/test/java/dev/toonformat/jtoon/conformance/ConformanceTest.java @@ -88,7 +88,7 @@ private EncodeOptions parseOptions(final JsonEncodeTestOptions options) { return EncodeOptions.DEFAULT; } - final int indent = options.indent() != null ? options.indent() : 2; + final int indent = options.indentSize() != null ? options.indentSize() : 2; Delimiter delimiter = Delimiter.COMMA; if (options.delimiter() != null) { @@ -190,7 +190,7 @@ private DecodeOptions parseOptions(final JsonDecodeTestOptions options) { return DecodeOptions.DEFAULT; } - final int indent = options.indent() != null ? options.indent() : 2; + final int indent = options.indentSize() != null ? options.indentSize() : 2; Delimiter delimiter = Delimiter.COMMA; if (options.delimiter() != null) { diff --git a/src/test/java/dev/toonformat/jtoon/conformance/model/JsonDecodeTestOptions.java b/src/test/java/dev/toonformat/jtoon/conformance/model/JsonDecodeTestOptions.java index ebc5feb3..7ebc9d31 100644 --- a/src/test/java/dev/toonformat/jtoon/conformance/model/JsonDecodeTestOptions.java +++ b/src/test/java/dev/toonformat/jtoon/conformance/model/JsonDecodeTestOptions.java @@ -1,7 +1,7 @@ package dev.toonformat.jtoon.conformance.model; public record JsonDecodeTestOptions( - Integer indent, + Integer indentSize, String delimiter, String lengthMarker, Boolean strict, diff --git a/src/test/java/dev/toonformat/jtoon/conformance/model/JsonEncodeTestOptions.java b/src/test/java/dev/toonformat/jtoon/conformance/model/JsonEncodeTestOptions.java index 4321d507..f952b499 100644 --- a/src/test/java/dev/toonformat/jtoon/conformance/model/JsonEncodeTestOptions.java +++ b/src/test/java/dev/toonformat/jtoon/conformance/model/JsonEncodeTestOptions.java @@ -1,7 +1,7 @@ package dev.toonformat.jtoon.conformance.model; public record JsonEncodeTestOptions( - Integer indent, + Integer indentSize, String delimiter, String lengthMarker, String keyFolding, diff --git a/src/test/java/dev/toonformat/jtoon/decoder/ArrayDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/ArrayDecoderTest.java index c32d76be..4487bf52 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/ArrayDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/ArrayDecoderTest.java @@ -16,6 +16,7 @@ class ArrayDecoderTest { private static final int EXPECTED_PARSE_COUNT = 3; private static final int MAX_ARRAY_SIZE = 10_000_000; + private static final int FIRST_DATA_LINE_INDEX = 3; private final DecodeContext context = new DecodeContext(); @@ -144,13 +145,13 @@ void expectsToExtractSlashFromDelimiter() { @DisplayName("Should validate array length") void validateArrayLength() { assertThrows(IllegalArgumentException.class, - () -> ArrayDecoder.validateArrayLength("[2]: 1,2,3", EXPECTED_PARSE_COUNT, MAX_ARRAY_SIZE)); + () -> ArrayDecoder.validateArrayLength("[2]: 1,2,3", EXPECTED_PARSE_COUNT, MAX_ARRAY_SIZE, true)); } @Test @DisplayName("Should validate array length") void validateArrayLengthWithoutException() { - assertDoesNotThrow(() -> ArrayDecoder.validateArrayLength("[2]: 1,2,3", 2, MAX_ARRAY_SIZE)); + assertDoesNotThrow(() -> ArrayDecoder.validateArrayLength("[2]: 1,2,3", 2, MAX_ARRAY_SIZE, true)); } @Test @@ -180,6 +181,144 @@ void shouldReturnEmptyListWhenInputIsEmpty() { assertTrue(result.isEmpty()); } + @Test + void shouldKeepQuotedDelimiterInsideValue() { + // When + final List result = ArrayDecoder.parseDelimitedValues("\"a,b\",c", Delimiter.COMMA); + + // Then + assertEquals(List.of("\"a,b\"", "c"), result); + } + + @Test + void shouldKeepEmptyValueInMiddle() { + // When + final List result = ArrayDecoder.parseDelimitedValues("a,,b", Delimiter.COMMA); + + // Then + assertEquals(List.of("a", "", "b"), result); + } + + @Test + void shouldTrimWhitespaceAroundDelimiters() { + // When + final List result = ArrayDecoder.parseDelimitedValues("a , b\t,\nc", Delimiter.COMMA); + + // Then + assertEquals(List.of("a", "b", "c"), result); + } + + @Test + void shouldRespectTabDelimiter() { + // When + final List result = ArrayDecoder.parseDelimitedValues("a\tb\tc", Delimiter.TAB); + + // Then + assertEquals(List.of("a", "b", "c"), result); + } + + @Test + void shouldRespectPipeDelimiter() { + // When + final List result = ArrayDecoder.parseDelimitedValues("a|b|c", Delimiter.PIPE); + + // Then + assertEquals(List.of("a", "b", "c"), result); + } + + @Test + void shouldKeepBackslashEscapedDelimiterInsideValue() { + // When + final List result = ArrayDecoder.parseDelimitedValues("a\\,b,c", Delimiter.COMMA); + + // Then + assertEquals(List.of("a\\,b", "c"), result); + } + + @Test + void shouldParseInlineArrayAfterColon() { + // Given + setUpContext("[3]: 1,2,3\nnext: value"); + + // When + final List result = ArrayDecoder.parseArray("[3]: 1,2,3", 0, context); + + // Then + assertEquals("[1, 2, 3]", result.toString()); + assertEquals(1, context.currentLine); + } + + @Test + void shouldRejectLeadingZeroLengthInStrictMode() { + // Given + setUpContext("[03]: 1,2,3"); + + // When / Then + final IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, + () -> ArrayDecoder.parseArray("[03]: 1,2,3", 0, context)); + assertEquals("Invalid array length with leading zeros: [03]", thrown.getMessage()); + } + + @Test + void shouldParseFirstDataLineAfterBlankLinesFollowingHeader() { + // Given + setUpContext("[2]:\n\n 1,2\n 3,4\nnext: value"); + + // When + final List result = ArrayDecoder.parseArray("[2]:", 0, context); + + // Then + assertEquals("[1, 2]", result.toString()); + assertEquals(FIRST_DATA_LINE_INDEX, context.currentLine); + } + + @Test + void shouldTreatShallowerNextLineAsEmptyArray() { + // Given + setUpContext("[0]:\nnext: value"); + + // When + final List result = ArrayDecoder.parseArray("[0]:", 0, context); + + // Then + assertTrue(result.isEmpty()); + } + + @Test + void shouldTreatBareBracketsAsEmptyArray() { + // Given + setUpContext("[]\nnext: value"); + + // When + final List result = ArrayDecoder.parseArray("[]", 0, context); + + // Then + assertTrue(result.isEmpty()); + } + + @Test + void shouldRejectInvalidHeaderInStrictMode() { + // Given + setUpContext("[x]"); + + // When / Then + assertThrows(IllegalArgumentException.class, () -> ArrayDecoder.parseArray("[x]", 0, context)); + } + + @Test + void shouldReturnEmptyListForInvalidHeaderInLenientMode() { + // Given + this.context.lines = "[x]".split("\n", -1); + this.context.options = DecodeOptions.withStrict(false); + this.context.delimiter = DecodeOptions.DEFAULT.delimiter(); + + // When + final List result = ArrayDecoder.parseArray("[x]", 0, context); + + // Then + assertTrue(result.isEmpty()); + } + @Test @DisplayName("extract length from the Header") void extractLengthFromHeader() throws Exception { diff --git a/src/test/java/dev/toonformat/jtoon/decoder/KeyDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/KeyDecoderTest.java index 314407c4..e8aced95 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/KeyDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/KeyDecoderTest.java @@ -11,6 +11,7 @@ import dev.toonformat.jtoon.DecodeOptions; import dev.toonformat.jtoon.Delimiter; import dev.toonformat.jtoon.PathExpansion; +import dev.toonformat.jtoon.util.Headers; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -152,8 +153,7 @@ void testThrowsIllegalArgumentExceptionWhenPathConflictsInStrictMode() { void testCallsExpandPathIntoMapWhenShouldExpandKeyTrue() { // Given final Map result = new LinkedHashMap<>(); - final String originalKey = "foo.bar"; - final String content = "foo.bar[#0]"; + final String content = "foo.bar[#0]:"; final int parentDepth = 0; final DecodeContext context = new DecodeContext(); @@ -165,7 +165,7 @@ void testCallsExpandPathIntoMapWhenShouldExpandKeyTrue() { final List expectedArray = Arrays.asList(1, 2, 3); // When - KeyDecoder.processKeyedArrayLine(result, content, originalKey, parentDepth, context); + KeyDecoder.processKeyedArrayLine(result, content, Headers.matchKeyedArrayHeader(content), parentDepth, context); // Then final Map expectedNestedMap = new LinkedHashMap<>(); @@ -180,7 +180,6 @@ void processKeyedArrayLine_givenBasicKeyedArray_whenProcessed_thenValueInMap() { // Given final Map result = new LinkedHashMap<>(); final String content = "tags[3]: a, b, c"; - final String originalKey = "tags"; final DecodeContext context = new DecodeContext(); context.options = new DecodeOptions(2, Delimiter.COMMA, true, PathExpansion.OFF, DecodeOptions.MAX_ALLOWED_DEPTH, DecodeOptions.DEFAULT_MAX_ARRAY_SIZE, @@ -188,7 +187,7 @@ void processKeyedArrayLine_givenBasicKeyedArray_whenProcessed_thenValueInMap() { context.delimiter = Delimiter.COMMA; // When - KeyDecoder.processKeyedArrayLine(result, content, originalKey, 0, context); + KeyDecoder.processKeyedArrayLine(result, content, Headers.matchKeyedArrayHeader(content), 0, context); // Then final List expected = Arrays.asList("a", "b", "c"); @@ -201,7 +200,6 @@ void processKeyedArrayLine_givenDottedKeyedArray_whenProcessed_thenNestedMapCont // Given final Map result = new LinkedHashMap<>(); final String content = "user.tags[2]: dev, test"; - final String originalKey = "user.tags"; final DecodeContext context = new DecodeContext(); context.options = new DecodeOptions(2, Delimiter.COMMA, true, PathExpansion.SAFE, DecodeOptions.MAX_ALLOWED_DEPTH, DecodeOptions.DEFAULT_MAX_ARRAY_SIZE, @@ -209,7 +207,7 @@ void processKeyedArrayLine_givenDottedKeyedArray_whenProcessed_thenNestedMapCont context.delimiter = Delimiter.COMMA; // When - KeyDecoder.processKeyedArrayLine(result, content, originalKey, 0, context); + KeyDecoder.processKeyedArrayLine(result, content, Headers.matchKeyedArrayHeader(content), 0, context); // Then assertTrue(result.containsKey("user")); @@ -226,7 +224,6 @@ void processKeyedArrayLine_givenExpansionConflictStrict_whenProcessed_thenThrows final Map result = new LinkedHashMap<>(); result.put("user", "not-a-map"); final String content = "user.tags[1]: dev"; - final String originalKey = "user.tags"; final DecodeContext context = new DecodeContext(); context.options = new DecodeOptions(2, Delimiter.COMMA, true, PathExpansion.SAFE, DecodeOptions.MAX_ALLOWED_DEPTH, DecodeOptions.DEFAULT_MAX_ARRAY_SIZE, @@ -235,7 +232,8 @@ void processKeyedArrayLine_givenExpansionConflictStrict_whenProcessed_thenThrows // When / Then final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, - () -> KeyDecoder.processKeyedArrayLine(result, content, originalKey, 0, context)); + () -> KeyDecoder.processKeyedArrayLine(result, content, + Headers.matchKeyedArrayHeader(content), 0, context)); assertTrue(ex.getMessage().contains("Path expansion conflict")); } @@ -262,6 +260,23 @@ void testEmptyValueCreatesLinkedHashMap() throws Exception { assertEquals(0, context.currentLine); } + @Test + void parseKeyValue_givenBracketPairValue_whenParsed_thenEmptyList() throws Exception { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"key: []"}; + context.currentLine = -1; + + // When + final Object result = invokePrivateStatic("parseKeyValue", + new Class[]{String.class, int.class, DecodeContext.class}, "[]", 0, context); + + // Then + assertInstanceOf(List.class, result, "Expected an empty list for [] value"); + assertTrue(((List) result).isEmpty(), "List should be empty"); + assertEquals(0, context.currentLine); + } + @Test void testExpandPathIntoMapCalledWhenShouldExpandKeyTrue() { // Given diff --git a/src/test/java/dev/toonformat/jtoon/decoder/KeyedObjectDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/KeyedObjectDecoderTest.java new file mode 100644 index 00000000..99a9656d --- /dev/null +++ b/src/test/java/dev/toonformat/jtoon/decoder/KeyedObjectDecoderTest.java @@ -0,0 +1,186 @@ +package dev.toonformat.jtoon.decoder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Map; +import dev.toonformat.jtoon.DecodeOptions; +import dev.toonformat.jtoon.util.Headers; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class KeyedObjectDecoderTest { + + private static final long PORT_VALUE = 8080L; + private static final int AFTER_ENTRIES_LINE_INDEX = 3; + + @Test + @DisplayName("Given keyed tabular object When parsed Then entry-keyed map returned") + @SuppressWarnings("unchecked") + void parseKeyedTabularObject_givenEntries_whenParsed_thenMap() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[2:]{host,port}:", + " alpha: a.example.com,8080", + " beta: b.example.com,9090"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When + final Map result = + KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context); + + // Then + assertEquals(2, result.size()); + final Map alpha = (Map) result.get("alpha"); + assertEquals("a.example.com", alpha.get("host")); + assertEquals(PORT_VALUE, alpha.get("port")); + assertEquals(AFTER_ENTRIES_LINE_INDEX, context.currentLine); + } + + @Test + @DisplayName("Given header without field list When parsed Then exception") + void parseKeyedTabularObject_givenMissingFieldsHeader_whenParsed_thenThrows() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[2:]:"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context)); + assertTrue(ex.getMessage().contains("requires a field list")); + } + + @Test + @DisplayName("Given inline content after keyed header When parsed Then exception") + void parseKeyedTabularObject_givenInlineContent_whenParsed_thenThrows() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[1:]{host}: x"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context)); + assertTrue(ex.getMessage().contains("Inline content after keyed header")); + } + + @Test + @DisplayName("Given entry count mismatch in strict mode When parsed Then exception") + void parseKeyedTabularObject_givenCountMismatchStrict_whenParsed_thenThrows() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[2:]{host,port}:", + " alpha: a.example.com,8080"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context)); + assertTrue(ex.getMessage().contains("does not match declared length")); + } + + @Test + @DisplayName("Given blank line inside object in strict mode When parsed Then exception") + void parseKeyedTabularObject_givenBlankLineInsideStrict_whenParsed_thenThrows() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[1:]{host}:", + " alpha: a.example.com", + "", + " beta: b.example.com"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context)); + assertTrue(ex.getMessage().contains("Blank line inside keyed object")); + } + + @Test + @DisplayName("Given blank line before first entry row When parsed Then accepted") + @SuppressWarnings("unchecked") + void parseKeyedTabularObject_givenBlankLineBeforeFirstRow_whenParsed_thenAccepted() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[1:]{host}:", + "", + " alpha: a.example.com"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When + final Map result = + KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context); + + // Then + assertEquals(1, result.size()); + assertInstanceOf(Map.class, result.get("alpha")); + } + + @Test + @DisplayName("Given over-indented line in strict mode When parsed Then exception") + void parseKeyedTabularObject_givenOverIndentedStrict_whenParsed_thenThrows() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[1:]{host}:", + " alpha: a.example.com", + " orphan"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context)); + assertTrue(ex.getMessage().contains("Over-indented line")); + } + + @Test + @DisplayName("Given entry without colon in lenient mode When parsed Then row skipped") + void parseKeyedTabularObject_givenMissingColonLenient_whenParsed_thenSkipped() { + // Given + final DecodeContext context = new DecodeContext(); + context.options = DecodeOptions.withStrict(false); + context.lines = new String[]{"servers[1:]{host}:", + " alpha: a.example.com", + " broken"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When + final Map result = + KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context); + + // Then + assertEquals(1, result.size()); + assertEquals(AFTER_ENTRIES_LINE_INDEX, context.currentLine); + } + + @Test + @DisplayName("Given shallower next line When parsed Then object ends") + void parseKeyedTabularObject_givenShallowerLine_whenParsed_thenEnds() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"servers[1:]{host}:", + " alpha: a.example.com", + "next: value"}; + context.currentLine = 0; + final Headers.KeyedHeaderMatch header = Headers.matchKeyedArrayHeader(context.lines[0]); + + // When + final Map result = + KeyedObjectDecoder.parseKeyedTabularObject(context.lines[0], header, 1, context); + + // Then + assertEquals(1, result.size()); + assertEquals(2, context.currentLine); + } +} diff --git a/src/test/java/dev/toonformat/jtoon/decoder/ListItemDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/ListItemDecoderTest.java index d1fd72fc..7e48db1f 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/ListItemDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/ListItemDecoderTest.java @@ -3,9 +3,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import dev.toonformat.jtoon.DecodeOptions; @@ -16,6 +18,8 @@ @Tag("unit") class ListItemDecoderTest { + private static final long SCALAR_ITEM_VALUE = 42L; + @Test @DisplayName("throws unsupported Operation Exception for calling the constructor") void throwsOnConstructor() throws NoSuchMethodException { @@ -86,7 +90,7 @@ void testParseListItemFields() throws Exception { final Map item = Map.of(line, testObject); final int depth = -2; final DecodeContext context = new DecodeContext(); - context.options = DecodeOptions.DEFAULT; + context.options = DecodeOptions.withStrict(false); context.lines = new String[] { line }; // When @@ -96,4 +100,153 @@ void testParseListItemFields() throws Exception { // Then assertEquals(1, context.currentLine); } + + @Test + @DisplayName("Given scalar item When parsed Then scalar returned and line advanced") + void parseListItem_givenScalarItem_whenParsed_thenScalar() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"- 42"}; + context.currentLine = 0; + + // When + final Object result = ListItemDecoder.parseListItem("- 42", 0, context); + + // Then + assertEquals(SCALAR_ITEM_VALUE, result); + assertEquals(1, context.currentLine); + } + + @Test + @DisplayName("Given empty item When parsed Then empty map returned") + void parseListItem_givenEmptyItem_whenParsed_thenEmptyMap() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"- "}; + context.currentLine = 0; + + // When + final Object result = ListItemDecoder.parseListItem("- ", 0, context); + + // Then + assertInstanceOf(Map.class, result); + assertTrue(((Map) result).isEmpty()); + assertEquals(1, context.currentLine); + } + + @Test + @DisplayName("Given standalone array item When parsed Then list returned") + void parseListItem_givenStandaloneArray_whenParsed_thenList() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"- [2]: 1,2"}; + context.currentLine = 0; + + // When + final Object result = ListItemDecoder.parseListItem("- [2]: 1,2", 0, context); + + // Then + assertEquals("[1, 2]", result.toString()); + assertEquals(1, context.currentLine); + } + + @Test + @DisplayName("Given keyless fields header item in strict mode When parsed Then exception") + void parseListItem_givenKeylessFieldsHeaderStrict_whenParsed_thenThrows() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"- [2]{x}: 1"}; + context.currentLine = 0; + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> ListItemDecoder.parseListItem("- [2]{x}: 1", 0, context)); + assertTrue(ex.getMessage().contains("Keyless array header with field list")); + } + + @Test + @DisplayName("Given keyed array item When parsed Then map with list returned") + void parseListItem_givenKeyedArray_whenParsed_thenMapWithList() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"- tags[2]: a,b"}; + context.currentLine = 0; + + // When + final Object result = ListItemDecoder.parseListItem("- tags[2]: a,b", 0, context); + + // Then + assertInstanceOf(Map.class, result); + assertEquals("[a, b]", ((Map) result).get("tags").toString()); + assertEquals(1, context.currentLine); + } + + @Test + @DisplayName("Given object item When parsed Then map returned") + void parseListItem_givenObjectItem_whenParsed_thenMap() { + // Given + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{"- key: value"}; + context.currentLine = 0; + + // When + final Object result = ListItemDecoder.parseListItem("- key: value", 0, context); + + // Then + assertInstanceOf(Map.class, result); + assertEquals("value", ((Map) result).get("key")); + assertEquals(1, context.currentLine); + } + + @Test + @DisplayName("Given over-indented field line in strict mode When parsed Then exception") + void parseListItemFields_givenOverIndentedStrict_whenParsed_thenThrows() throws Exception { + // Given + final Map item = new LinkedHashMap<>(); + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{" - item", " orphan", " - next"}; + context.currentLine = 1; + + // When / Then + final InvocationTargetException ex = assertThrows(InvocationTargetException.class, + () -> invokePrivateStatic("parseListItemFields", + new Class[]{Map.class, int.class, DecodeContext.class}, item, 0, context)); + assertInstanceOf(IllegalArgumentException.class, ex.getCause()); + } + + @Test + @DisplayName("Given over-indented field line in lenient mode When parsed Then line skipped") + void parseListItemFields_givenOverIndentedLenient_whenParsed_thenSkipped() throws Exception { + // Given + final Map item = new LinkedHashMap<>(); + final DecodeContext context = new DecodeContext(); + context.options = DecodeOptions.withStrict(false); + context.lines = new String[]{" - item", " orphan", " - next"}; + context.currentLine = 1; + + // When + invokePrivateStatic("parseListItemFields", + new Class[]{Map.class, int.class, DecodeContext.class}, item, 0, context); + + // Then + assertEquals(2, context.currentLine); + } + + @Test + @DisplayName("Given sibling field lines When parsed Then fields added to item") + void parseListItemFields_givenFieldLines_whenParsed_thenFieldsAdded() throws Exception { + // Given + final Map item = new LinkedHashMap<>(); + final DecodeContext context = new DecodeContext(); + context.lines = new String[]{" - item", " extra: 1", " - next"}; + context.currentLine = 1; + + // When + invokePrivateStatic("parseListItemFields", + new Class[]{Map.class, int.class, DecodeContext.class}, item, 0, context); + + // Then + assertEquals(1L, item.get("extra")); + assertEquals(2, context.currentLine); + } } diff --git a/src/test/java/dev/toonformat/jtoon/decoder/ObjectDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/ObjectDecoderTest.java index 96db38ba..04c5dbf6 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/ObjectDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/ObjectDecoderTest.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; import dev.toonformat.jtoon.DecodeOptions; +import dev.toonformat.jtoon.util.Headers; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; @@ -107,7 +108,7 @@ void parseNestedObject_basic() { } @Test - @DisplayName("GIVEN deeper indentation WHEN child is not direct child THEN skip line") + @DisplayName("GIVEN deeper indentation WHEN child is not direct child THEN skip line in lenient mode") void parseNestedObject_skips_invalid_depth() { // Given setUpContext(""" @@ -115,7 +116,7 @@ void parseNestedObject_skips_invalid_depth() { tooDeep: X child: OK """); - + context.options = DecodeOptions.withStrict(false); context.currentLine = 1; // When @@ -152,6 +153,7 @@ void parseRootObjectFields_basic() { b: 20 nested: IGNORE """); + context.options = DecodeOptions.withStrict(false); final Map root = new LinkedHashMap<>(); // When @@ -322,6 +324,72 @@ void parseFieldValue_empty_no_nestedButBigCurrentLine() { assertInstanceOf(Map.class, parseFieldValue); assertEquals(offset + 1, context.currentLine); } + + @Test + @DisplayName("GIVEN inline value + deeper line + strict => over-indented exception") + void parseFieldValue_inlineValueWithDeeperLineThrowsInStrictMode() { + // Given + setUpContext(""" + key: 15 + orphan + """); + context.currentLine = 0; + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> ObjectDecoder.parseFieldValue("15", 0, context)); + assertTrue(ex.getMessage().contains("Over-indented line")); + } + + @Test + @DisplayName("GIVEN inline value + deeper line + lenient => value kept, orphan lines skipped") + void parseFieldValue_inlineValueWithDeeperLineSkippedInLenientMode() { + // Given + setUpContext(""" + key: 15 + orphan + """); + context.options = DecodeOptions.withStrict(false); + context.currentLine = 0; + + // When + final Object parseFieldValue = ObjectDecoder.parseFieldValue("15", 0, context); + + // Then + assertEquals(SCALAR_PARSE_VALUE, parseFieldValue); + assertEquals(2, context.currentLine); + } + + @Test + @DisplayName("GIVEN bracket pair value => parsed as string, not empty list") + void parseFieldValue_bracketPairStaysString() { + // Given + setUpContext("key: []"); + context.currentLine = 0; + + // When + final Object parseFieldValue = ObjectDecoder.parseFieldValue("[]", 0, context); + + // Then + assertEquals("[]", parseFieldValue); + assertEquals(1, context.currentLine); + } + + @Test + @DisplayName("GIVEN blank value on last line => empty map") + void parseFieldValue_blankValueOnLastLineBecomesEmptyMap() { + // Given + setUpContext("key: "); + context.currentLine = 0; + + // When + final Object parseFieldValue = ObjectDecoder.parseFieldValue("", 0, context); + + // Then + assertInstanceOf(Map.class, parseFieldValue); + assertTrue(((Map) parseFieldValue).isEmpty()); + assertEquals(1, context.currentLine); + } } @Nested @@ -420,8 +488,8 @@ void testExpandPathIntoMapCalledForDottedKey() throws Exception { // When invokePrivateStatic( "processRootKeyedArrayLine", - new Class[]{Map.class, String.class, String.class, int.class, DecodeContext.class}, - objectMap, content, "user.name", depth, context); + new Class[]{Map.class, String.class, Headers.KeyedHeaderMatch.class, int.class, DecodeContext.class}, + objectMap, content, Headers.matchKeyedArrayHeader(content), depth, context); // Then assertTrue(objectMap.containsKey("user.name")); diff --git a/src/test/java/dev/toonformat/jtoon/decoder/TabularArrayDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/TabularArrayDecoderTest.java index 7ac78ec8..6c866e1b 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/TabularArrayDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/TabularArrayDecoderTest.java @@ -19,6 +19,8 @@ @Tag("unit") class TabularArrayDecoderTest { + private static final int SIMPLE_FIELD_COUNT = 3; + private final DecodeContext context = new DecodeContext(); DecodeOptions before; @@ -92,12 +94,13 @@ void inCaseOfMismatchInDelimiter_ThrowAnException() { } @Test - @DisplayName("processTabularRow: deeper-than-expected line is skipped (else-if branch)") + @DisplayName("processTabularRow: deeper-than-expected line is skipped in lenient mode (else-if branch)") void processTabularRow_skipsDeeperIndentedLine() { // Given final String toon = "[2]{id,name}:\n 1,Ada\n nested: true\n 2,Bob"; setUpContext(toon); + context.options = DecodeOptions.withStrict(false); // When final List result = TabularArrayDecoder.parseTabularArray(toon, 0, @@ -345,8 +348,143 @@ void testDisambiguation_PipeDelimiter_continuesRow() throws Exception { } @Test - void testParseTabularArray_ReturnsEmptyList_WhenHeaderDoesNotMatchPattern() { + @DisplayName("Parse simple field list into leaf nodes") + void parseTabularKeys_givenSimpleList_thenLeafNodes() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("a,b,c", Delimiter.COMMA, ctx); + + // Then + assertEquals(SIMPLE_FIELD_COUNT, fields.size()); + assertEquals("a", fields.get(0).name()); + assertEquals("b", fields.get(1).name()); + assertEquals("c", fields.get(2).name()); + assertTrue(fields.get(0).children().isEmpty()); + } + + @Test + @DisplayName("Parse backslash-escaped backslash inside a field name") + void parseTabularKeys_givenEscapedBackslash_thenSingleField() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("a\\\\b,c", Delimiter.COMMA, ctx); + + // Then + assertEquals(2, fields.size()); + assertEquals("a\\b", fields.get(0).name()); + assertEquals("c", fields.get(1).name()); + } + + @Test + @DisplayName("Parse quoted field name preserving delimiter characters") + void parseTabularKeys_givenQuotedName_thenDelimiterPreserved() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("\"a,b\",c", Delimiter.COMMA, ctx); + + // Then + assertEquals(2, fields.size()); + assertEquals("a,b", fields.get(0).name()); + assertEquals("c", fields.get(1).name()); + } + + @Test + @DisplayName("Parse nested field group into parent field with children") + void parseTabularKeys_givenNestedGroup_thenParentWithChildren() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("a{b,c},d", Delimiter.COMMA, ctx); + + // Then + assertEquals(2, fields.size()); + assertEquals("a", fields.get(0).name()); + assertEquals(2, fields.get(0).children().size()); + assertEquals("b", fields.get(0).children().get(0).name()); + assertEquals("c", fields.get(0).children().get(1).name()); + assertEquals("d", fields.get(1).name()); + } + + @Test + @DisplayName("Throw on unbalanced braces in strict mode") + void parseTabularKeys_givenUnbalancedStrict_thenThrows() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When / Then + final IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> TabularArrayDecoder.parseTabularKeys("a{b,c", Delimiter.COMMA, ctx)); + assertTrue(ex.getMessage().contains("Unbalanced braces")); + } + + @Test + @DisplayName("Skip unbalanced group in lenient mode and keep parsed fields") + void parseTabularKeys_givenUnbalancedLenient_thenPartialFields() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.withStrict(false); + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("a{b,c", Delimiter.COMMA, ctx); + + // Then + assertEquals(1, fields.size()); + assertEquals("a", fields.get(0).name()); + } + + @Test + @DisplayName("Skip whitespace after delimiter in field list") + void parseTabularKeys_givenWhitespaceAfterDelimiter_thenTrimmedFields() { // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("a , b", Delimiter.COMMA, ctx); + + // Then + assertEquals(2, fields.size()); + assertEquals("a", fields.get(0).name()); + assertEquals("b", fields.get(1).name()); + } + + @Test + @DisplayName("Parse field list with pipe delimiter") + void parseTabularKeys_givenPipeDelimiter_thenFields() { + // Given + final DecodeContext ctx = new DecodeContext(); + ctx.options = DecodeOptions.DEFAULT; + + // When + final List fields = + TabularArrayDecoder.parseTabularKeys("x|y", Delimiter.PIPE, ctx); + + // Then + assertEquals(2, fields.size()); + assertEquals("x", fields.get(0).name()); + assertEquals("y", fields.get(1).name()); + } + + @Test + void testParseTabularArray_ReturnsEmptyList_WhenHeaderDoesNotMatchPattern() { // Given context.options = new DecodeOptions(2, Delimiter.COMMA, false, PathExpansion.OFF, DecodeOptions.MAX_ALLOWED_DEPTH, DecodeOptions.DEFAULT_MAX_ARRAY_SIZE, DecodeOptions.DEFAULT_MAX_STRING_LENGTH); diff --git a/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java b/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java index 8f7e8e32..20a37db9 100644 --- a/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/decoder/ValueDecoderTest.java @@ -77,6 +77,115 @@ void decode_returnsEmptyMap_whenProcessedIsEmpty() { assertTrue(((LinkedHashMap) result).isEmpty(), "Map must be empty"); } + @Test + @DisplayName("strips a single leading byte-order mark before decoding") + void decode_stripsLeadingByteOrderMark() { + // Given + final String input = Character.toString(0xFEFF) + "name: Ada"; + + // When + final Object result = ValueDecoder.decode(input, DecodeOptions.DEFAULT); + + // Then + assertEquals("{name=Ada}", result.toString()); + } + + @Test + @DisplayName("keeps a byte-order mark that is not the first character as content") + void decode_keepsNonLeadingByteOrderMarkAsContent() { + // Given + final String bom = Character.toString(0xFEFF); + final String input = "name: " + bom + "Ada"; + + // When + final Object result = ValueDecoder.decode(input, DecodeOptions.DEFAULT); + + // Then + assertEquals("{name=" + bom + "Ada}", result.toString()); + } + + @Test + @DisplayName("discards full-line comment lines before structural parsing") + void decode_discardsFullLineComments() { + // Given + final String input = "# a comment\nname: Ada"; + + // When + final Object result = ValueDecoder.decode(input, DecodeOptions.DEFAULT); + + // Then + assertEquals("{name=Ada}", result.toString()); + } + + @Test + @DisplayName("treats a hash not at line start as data") + void decode_hashInsideLineIsData() { + // When + final Object result = ValueDecoder.decode("name: a#b", DecodeOptions.DEFAULT); + + // Then + assertEquals("{name=a#b}", result.toString()); + } + + @Test + @DisplayName("decodes an empty array literal as an empty list") + void decode_emptyArrayLiteral() { + // When + final Object result = ValueDecoder.decode("[]", DecodeOptions.DEFAULT); + + // Then + assertEquals("[]", result.toString()); + } + + @Test + @DisplayName("decodes a keyless root array header as an array") + void decode_keylessRootArrayHeader() { + // When + final Object result = ValueDecoder.decode("[2]: 1,2", DecodeOptions.DEFAULT); + + // Then + assertEquals("[1, 2]", result.toString()); + } + + @Test + @DisplayName("decodes a keyed tabular root header as a keyed object") + void decode_keyedTabularRootHeader() { + // Given + final String input = "servers[1:]{host,port}:\n alpha: a.example.com,8080"; + + // When + final Object result = ValueDecoder.decode(input, DecodeOptions.DEFAULT); + + // Then + assertEquals("{servers={alpha={host=a.example.com, port=8080}}}", result.toString()); + } + + @Test + @DisplayName("parses a bare scalar at the root") + void decode_bareScalarRoot() { + // When + final Object result = ValueDecoder.decode("42", DecodeOptions.DEFAULT); + + // Then + assertEquals("42", result.toString()); + } + + @Test + @DisplayName("rejects stray unquoted brackets in a root key in strict mode") + void decode_strict_rejectsStrayBracketsInRootKey() { + // When / Then + assertThrows(IllegalArgumentException.class, + () -> ValueDecoder.decode("foo[1][bar]: x", DecodeOptions.DEFAULT)); + } + + @Test + @DisplayName("rejects unquoted brackets without a valid header in strict mode") + void decode_strict_rejectsBracketsWithoutValidHeader() { + // When / Then + assertThrows(IllegalArgumentException.class, + () -> ValueDecoder.decode("items[2]{id,name}", DecodeOptions.DEFAULT)); + } + @Test @DisplayName("Should parse TOON format primitive array to JSON") void parsePrimitiveArray() { diff --git a/src/test/java/dev/toonformat/jtoon/encoder/HeaderFormatterTest.java b/src/test/java/dev/toonformat/jtoon/encoder/HeaderFormatterTest.java index 15858b6c..6fabcfb0 100644 --- a/src/test/java/dev/toonformat/jtoon/encoder/HeaderFormatterTest.java +++ b/src/test/java/dev/toonformat/jtoon/encoder/HeaderFormatterTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.*; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.List; import java.util.stream.Stream; import dev.toonformat.jtoon.Delimiter; @@ -21,6 +22,10 @@ @Tag("unit") public class HeaderFormatterTest { + private static List leafFields(final String... names) { + return Arrays.stream(names).map(TabularField::leaf).toList(); + } + @Nested @DisplayName("Simple Array Headers") class SimpleArrayHeaders { @@ -74,7 +79,7 @@ class TabularHeaders { @DisplayName("should format tabular header with fields") void testTabularHeader() { // Given - final List fields = List.of("id", "name", "age"); + final List fields = leafFields("id", "name", "age"); // When final String result = HeaderFormatter.format(2, "users", fields, Delimiter.COMMA.toString(), false); @@ -87,7 +92,7 @@ void testTabularHeader() { @DisplayName("should format tabular header with single field") void testSingleField() { // Given - final List fields = List.of("value"); + final List fields = leafFields("value"); // When final String result = HeaderFormatter.format(5, "data", fields, Delimiter.COMMA.toString(), false); @@ -100,7 +105,7 @@ void testSingleField() { @DisplayName("should format tabular header without key") void testTabularWithoutKey() { // Given - final List fields = List.of("x", "y"); + final List fields = leafFields("x", "y"); // When final String result = HeaderFormatter.format(10, null, fields, Delimiter.COMMA.toString(), false); @@ -113,7 +118,7 @@ void testTabularWithoutKey() { @DisplayName("should format empty tabular header (no fields)") void testEmptyFields() { // Given - final List fields = List.of(); + final List fields = leafFields(); // When final String result = HeaderFormatter.format(3, "items", fields, Delimiter.COMMA.toString(), false); @@ -126,7 +131,7 @@ void testEmptyFields() { @DisplayName("should format tabular header with length marker") void testTabularWithLengthMarker() { // Given - final List fields = List.of("id", "name"); + final List fields = leafFields("id", "name"); // When final String result = HeaderFormatter.format(2, "users", fields, Delimiter.COMMA.toString(), true); @@ -145,7 +150,7 @@ class DelimiterVariations { @DisplayName("should format with different delimiters") void testDelimiterFormatting(final String delimiterName, final String delimiter, final String expected) { // Given - final List fields = List.of("a", "b", "c"); + final List fields = leafFields("a", "b", "c"); // When final String result = HeaderFormatter.format(3, "data", fields, delimiter, false); @@ -185,7 +190,7 @@ void testArrayWithTabDelimiter() { @DisplayName("should format with pipe delimiter and length marker") void testPipeWithLengthMarker() { // Given - final List fields = List.of("x", "y"); + final List fields = leafFields("x", "y"); // When final String result = HeaderFormatter.format(2, "points", fields, Delimiter.PIPE.toString(), true); @@ -233,7 +238,7 @@ void testSimpleKey() { @DisplayName("should quote field names with special characters") void testFieldQuoting() { // Given - final List fields = List.of("first name", "last name"); + final List fields = leafFields("first name", "last name"); // When final String result = HeaderFormatter.format(2, "users", fields, Delimiter.COMMA.toString(), false); @@ -246,7 +251,7 @@ void testFieldQuoting() { @DisplayName("should handle mix of quoted and unquoted field names") void testMixedFieldQuoting() { // Given - final List fields = List.of("id", "full name", "age"); + final List fields = leafFields("id", "full name", "age"); // When final String result = HeaderFormatter.format(2, "users", fields, Delimiter.COMMA.toString(), false); @@ -265,7 +270,7 @@ class RecordBasedFormat { void testRecordFormat() { // Given final HeaderFormatter.HeaderConfig config = new HeaderFormatter.HeaderConfig( - 3, "items", List.of("id", "name"), Delimiter.COMMA.toString(), false); + 3, "items", leafFields("id", "name"), Delimiter.COMMA.toString(), false); // When final String result = HeaderFormatter.format(config); @@ -293,7 +298,7 @@ void testRecordWithNullKey() { void testRecordWithPipeDelimiter() { // Given final HeaderFormatter.HeaderConfig config = new HeaderFormatter.HeaderConfig( - 2, "data", List.of("x", "y"), Delimiter.PIPE.toString(), true); + 2, "data", leafFields("x", "y"), Delimiter.PIPE.toString(), true); // When final String result = HeaderFormatter.format(config); @@ -320,7 +325,7 @@ void testLargeLength() { @DisplayName("should handle zero length with fields") void testZeroLengthWithFields() { // Given - final List fields = List.of("id", "name"); + final List fields = leafFields("id", "name"); // When final String result = HeaderFormatter.format(0, "empty", fields, Delimiter.COMMA.toString(), false); @@ -333,7 +338,7 @@ void testZeroLengthWithFields() { @DisplayName("should handle many fields") void testManyFields() { // Given - final List fields = List.of("f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10"); + final List fields = leafFields("f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10"); // When final String result = HeaderFormatter.format(1, "data", fields, Delimiter.COMMA.toString(), false); @@ -361,7 +366,7 @@ class RealWorldExamples { @DisplayName("should format GitHub repositories header") void testGitHubRepos() { // Given - final List fields = List.of("id", "name", "stars", "forks"); + final List fields = leafFields("id", "name", "stars", "forks"); // When final String result = HeaderFormatter.format(100, "repositories", fields, @@ -375,7 +380,7 @@ void testGitHubRepos() { @DisplayName("should format analytics metrics header") void testAnalyticsMetrics() { // Given - final List fields = List.of("date", "views", "clicks", "conversions", "revenue"); + final List fields = leafFields("date", "views", "clicks", "conversions", "revenue"); // When final String result = HeaderFormatter.format(180, "metrics", fields, ",", false); @@ -388,7 +393,7 @@ void testAnalyticsMetrics() { @DisplayName("should format employee records with tab delimiter") void testEmployeeRecords() { // Given - final List fields = List.of("id", "name", "department", "salary"); + final List fields = leafFields("id", "name", "department", "salary"); // When final String result = HeaderFormatter.format(50, "employees", fields, Delimiter.TAB.toString(), false); @@ -401,7 +406,7 @@ void testEmployeeRecords() { @DisplayName("should format nested array in list item") void testNestedArray() { // Given - final List fields = List.of("sku", "qty", "price"); + final List fields = leafFields("sku", "qty", "price"); // When final String result = HeaderFormatter.format(3, "items", fields, Delimiter.COMMA.toString(), false); diff --git a/src/test/java/dev/toonformat/jtoon/encoder/PrimitiveEncoderTest.java b/src/test/java/dev/toonformat/jtoon/encoder/PrimitiveEncoderTest.java index caf53da1..c697c9dd 100644 --- a/src/test/java/dev/toonformat/jtoon/encoder/PrimitiveEncoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/encoder/PrimitiveEncoderTest.java @@ -535,7 +535,7 @@ void testSimpleHeader() { @DisplayName("should format tabular header") void testTabularHeader() { // Given - final List fields = List.of("id", "name"); + final List fields = List.of(TabularField.leaf("id"), TabularField.leaf("name")); // When final String result = PrimitiveEncoder.formatHeader(3, "users", fields, Delimiter.COMMA.toString(), false); @@ -558,7 +558,7 @@ void testWithLengthMarker() { @DisplayName("should format header with pipe delimiter") void testPipeDelimiter() { // Given - final List fields = List.of("x", "y"); + final List fields = List.of(TabularField.leaf("x"), TabularField.leaf("y")); // When final String result = PrimitiveEncoder.formatHeader(2, "points", fields, Delimiter.PIPE.toString(), false); diff --git a/src/test/java/dev/toonformat/jtoon/encoder/TabularArrayEncoderTest.java b/src/test/java/dev/toonformat/jtoon/encoder/TabularArrayEncoderTest.java index 0af47538..ccf769ee 100644 --- a/src/test/java/dev/toonformat/jtoon/encoder/TabularArrayEncoderTest.java +++ b/src/test/java/dev/toonformat/jtoon/encoder/TabularArrayEncoderTest.java @@ -39,7 +39,7 @@ void givenEmptyArray_whenDetectHeader_thenReturnsEmpty() { final ArrayNode rows = jsonNodeFactory.arrayNode(); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -51,7 +51,7 @@ void givenFirstRowNotObject_whenDetectHeader_thenReturnsEmpty() { final ArrayNode rows = jsonNodeFactory.arrayNode().add(1).add(2); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -64,7 +64,7 @@ void givenFirstObjectHasNoKeys_whenDetectHeader_thenReturnsEmpty() { rows.add(jsonNodeFactory.objectNode()); // empty object // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -83,7 +83,7 @@ void givenMismatchedKeyCount_whenDetectHeader_thenReturnsEmpty() { final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -104,7 +104,7 @@ void givenMissingHeaderKeyInLaterRow_whenDetectHeader_thenReturnsEmpty() { final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -124,7 +124,7 @@ void givenNonPrimitiveValue_whenDetectHeader_thenReturnsEmpty() { final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -144,10 +144,10 @@ void givenUniformObjectsDifferentOrder_whenDetectHeader_thenReturnsHeaderKeys() final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then - assertEquals(List.of("id", "name"), header); + assertEquals(List.of("id", "name"), header.stream().map(TabularField::name).toList()); } @Test @@ -162,7 +162,7 @@ void givenUniformObjects_whenEncodeArrayAsTabular_thenWritesHeaderAndRows() { b.put("name", "Bob"); final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); final LineWriter writer = new LineWriter(options.indent()); // When @@ -192,7 +192,7 @@ void givenHeaderAndRows_whenWriteTabularRows_thenWritesValuesWithIndent() { b.put("y", nodeBy); final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); - final List header = List.of("x", "y"); + final List header = List.of(TabularField.leaf("x"), TabularField.leaf("y")); final LineWriter writer = new LineWriter(options.indent()); // When @@ -219,10 +219,10 @@ void givenUniformObjects_whenDetectHeader_thenReturnsUnmodifiableList() { final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then - assertThrows(UnsupportedOperationException.class, () -> header.add("extra")); + assertThrows(UnsupportedOperationException.class, () -> header.add(TabularField.leaf("extra"))); } @Test @@ -231,7 +231,7 @@ void testDetectTabularHeaderWithEmptyRow() { final ArrayNode rows = jsonNodeFactory.arrayNode(); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -244,7 +244,7 @@ void testDetectTabularHeaderWithNoneObjectAsFirstItem() { rows.add(1); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -258,7 +258,7 @@ void testDetectTabularHeaderWithEmptyObject() { rows.add(a); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -272,7 +272,7 @@ void testDetectTabularHeaderWithSecondItemIsNotAnObject() { rows.add(a).add(1); // When - final List header = TabularArrayEncoder.detectTabularHeader(rows); + final List header = TabularArrayEncoder.detectTabularHeader(rows); // Then assertTrue(header.isEmpty()); @@ -292,7 +292,7 @@ void testDetectTabularHeaderWithUnevenObjectInTheList() { b.put("x", objBx); final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); - final List header = List.of("x", "y"); + final List header = List.of(TabularField.leaf("x"), TabularField.leaf("y")); final LineWriter writer = new LineWriter(options.indent()); // When @@ -321,7 +321,7 @@ void testDetectTabularHeaderWithUnevenObjectArrayMixInTheList() { b.add(mixArr2); final ArrayNode rows = jsonNodeFactory.arrayNode().add(a).add(b); - final List header = List.of("x", "y"); + final List header = List.of(TabularField.leaf("x"), TabularField.leaf("y")); final LineWriter writer = new LineWriter(options.indent()); // When diff --git a/src/test/java/dev/toonformat/jtoon/util/HeadersTest.java b/src/test/java/dev/toonformat/jtoon/util/HeadersTest.java index 1b2edbff..fde54d82 100644 --- a/src/test/java/dev/toonformat/jtoon/util/HeadersTest.java +++ b/src/test/java/dev/toonformat/jtoon/util/HeadersTest.java @@ -12,6 +12,8 @@ @DisplayName("Headers") class HeadersTest { + private static final long TRIPLE_LENGTH = 3L; + @Test @DisplayName("constructor throws UnsupportedOperationException") void constructorThrowsException() throws Exception { @@ -69,4 +71,131 @@ void keyedArrayPatternNoMatch() { // Negative length assertFalse(Headers.KEYED_ARRAY_PATTERN.matcher("items[-1]:").matches()); } + + @Test + @DisplayName("matchKeyedArrayHeader scans key, bracket and field spec segments") + void matchKeyedArrayHeader_givenFullHeader_thenSegments() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeyedArrayHeader("items[2]{id,name}:"); + + // Then + assertNotNull(match); + assertEquals("items", match.key()); + assertEquals(2L, match.declaredLength()); + assertFalse(match.keyed()); + assertNull(match.delimiter()); + assertTrue(match.fieldsStart() > match.keyEnd()); + assertTrue(match.headerEnd() > match.fieldsStart()); + } + + @Test + @DisplayName("matchKeyedArrayHeader scans keyed marker and delimiter declarations") + void matchKeyedArrayHeader_givenKeyedMarkerAndDelimiter_thenDeclared() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeyedArrayHeader("items[2:|]:a,b"); + + // Then + assertNotNull(match); + assertEquals(2L, match.declaredLength()); + assertTrue(match.keyed()); + assertEquals('|', match.delimiter().charValue()); + } + + @Test + @DisplayName("matchKeyedArrayHeader scans hash marker and field spec") + void matchKeyedArrayHeader_givenHashMarker_thenLengthDeclared() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeyedArrayHeader("items[#2]{a,b}:"); + + // Then + assertNotNull(match); + assertEquals(2L, match.declaredLength()); + assertFalse(match.keyed()); + assertEquals("items", match.key()); + } + + @Test + @DisplayName("matchKeyedArrayHeader scans quoted keys with spaces") + void matchKeyedArrayHeader_givenQuotedKey_thenKey() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeyedArrayHeader("\"my items\"[3]:"); + + // Then + assertNotNull(match); + assertEquals("\"my items\"", match.key()); + assertEquals(TRIPLE_LENGTH, match.declaredLength()); + } + + @Test + @DisplayName("matchKeyedArrayHeader scans quoted keys with escaped quotes") + void matchKeyedArrayHeader_givenEscapedQuoteKey_thenKey() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeyedArrayHeader("\"name\\\"with\\\"quotes\"[3]:"); + + // Then + assertNotNull(match); + assertEquals("\"name\\\"with\\\"quotes\"", match.key()); + } + + @Test + @DisplayName("matchKeyedArrayHeader scans field specs with nested braces") + void matchKeyedArrayHeader_givenNestedFieldSpec_thenMatch() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeyedArrayHeader("geo[2]{point{lat,lon}}:"); + + // Then + assertNotNull(match); + assertEquals("geo", match.key()); + assertTrue(match.fieldsStart() > match.keyEnd()); + assertTrue(match.headerEnd() > match.fieldsStart()); + } + + @Test + @DisplayName("matchKeylessKeyedHeader scans plain keyless tabular header") + void matchKeylessKeyedHeader_givenPlainHeader_thenNotKeyed() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeylessKeyedHeader("[2]{id,name}:"); + + // Then + assertNotNull(match); + assertEquals("", match.key()); + assertFalse(match.keyed()); + assertTrue(match.fieldsStart() > 0); + } + + @Test + @DisplayName("matchKeylessKeyedHeader scans keyed keyless header") + void matchKeylessKeyedHeader_givenKeyedMarker_thenKeyed() { + // Given / When + final Headers.KeyedHeaderMatch match = Headers.matchKeylessKeyedHeader("[2:]{id,name}:"); + + // Then + assertNotNull(match); + assertTrue(match.keyed()); + assertTrue(match.fieldsStart() > 0); + } + + @Test + @DisplayName("matchKeyedArrayHeader rejects malformed headers") + void matchKeyedArrayHeader_givenMalformed_thenNull() { + // Missing trailing colon + assertNull(Headers.matchKeyedArrayHeader("items[3]")); + // Missing bracket segment + assertNull(Headers.matchKeyedArrayHeader("items:")); + // Empty bracket segment + assertNull(Headers.matchKeyedArrayHeader("items[]:")); + // Unterminated quoted key + assertNull(Headers.matchKeyedArrayHeader("\"abc[3]:")); + // Unbalanced field spec braces + assertNull(Headers.matchKeyedArrayHeader("items[2]{a,b:")); + } + + @Test + @DisplayName("matchKeylessKeyedHeader rejects malformed headers") + void matchKeylessKeyedHeader_givenMalformed_thenNull() { + // Non-digit length + assertNull(Headers.matchKeylessKeyedHeader("[x]:")); + // Missing trailing colon + assertNull(Headers.matchKeylessKeyedHeader("[2]{a,b}")); + } } diff --git a/src/test/resources/conformance/decode/arrays-nested.json b/src/test/resources/conformance/decode/arrays-nested.json index f66e6de3..57ffe906 100644 --- a/src/test/resources/conformance/decode/arrays-nested.json +++ b/src/test/resources/conformance/decode/arrays-nested.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "decode", - "description": "Nested and mixed array decoding - list format, arrays of arrays, root arrays, mixed types", + "description": "Nested and mixed array decoding – list form, arrays of arrays, root arrays, mixed types", "tests": [ { "name": "parses list arrays for non-uniform objects", @@ -84,7 +84,7 @@ "specSection": "10" }, { - "name": "parses objects containing arrays (including empty arrays) in list format", + "name": "parses objects containing arrays (including empty arrays) in list form", "input": "items[1]:\n - name: Ada\n data[0]:", "expected": { "items": [ @@ -142,19 +142,19 @@ "specSection": "9.1" }, { - "name": "parses root-level array of uniform objects in tabular format", + "name": "parses root-level array of uniform objects in tabular form", "input": "[2]{id}:\n 1\n 2", "expected": [{ "id": 1 }, { "id": 2 }], "specSection": "9.3" }, { - "name": "parses root-level array of non-uniform objects in list format", + "name": "parses root-level array of non-uniform objects in list form", "input": "[2]:\n - id: 1\n - id: 2\n name: Ada", "expected": [{ "id": 1 }, { "id": 2, "name": "Ada" }], "specSection": "9.4" }, { - "name": "parses root-level array mixing primitive, object, and array of objects in list format", + "name": "parses root-level array mixing primitive, object, and array of objects in list form", "input": "[3]:\n - summary\n - id: 1\n name: Ada\n - [2]:\n - id: 2\n - status: draft", "expected": ["summary", { "id": 1, "name": "Ada" }, [{ "id": 2 }, { "status": "draft" }]], "specSection": "9.4" @@ -186,7 +186,7 @@ "specSection": "8" }, { - "name": "parses arrays mixing primitives, objects, and strings in list format", + "name": "parses arrays mixing primitives, objects, and strings in list form", "input": "items[3]:\n - 1\n - a: 1\n - text", "expected": { "items": [1, { "a": 1 }, "text"] @@ -202,7 +202,7 @@ "specSection": "9.4" }, { - "name": "parses quoted key with list array format", + "name": "parses quoted key with list-form array", "input": "\"x-items\"[2]:\n - id: 1\n - id: 2", "expected": { "x-items": [ @@ -211,6 +211,39 @@ ] }, "specSection": "9.4" + }, + { + "name": "accepts bare bracket pair as empty inner array list item", + "input": "items[2]:\n - []\n - [1]: x", + "expected": { + "items": [[], ["x"]] + }, + "specSection": "9.2", + "note": "Decode-side acceptance mirrors §9.1's key: [] form; encoders still emit - [0]:" + }, + { + "name": "keeps every list item when the count mismatches in non-strict mode", + "input": "a[1]:\n - 1\n - 2", + "expected": { + "a": [ + 1, + 2 + ] + }, + "options": { + "strict": false + }, + "specSection": "14.1", + "note": "A declared [N] never truncates a scope", + "minSpecVersion": "4.1" + }, + { + "name": "throws on a non-list-item line at item depth", + "input": "items[2]:\n - a\n x: 1", + "expected": null, + "shouldError": true, + "specSection": "9.4", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/decode/arrays-primitive.json b/src/test/resources/conformance/decode/arrays-primitive.json index 63ff03fe..e09b87ff 100644 --- a/src/test/resources/conformance/decode/arrays-primitive.json +++ b/src/test/resources/conformance/decode/arrays-primitive.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "decode", - "description": "Primitive array decoding - inline arrays of strings, numbers, booleans, quoted strings", + "description": "Primitive array decoding – inline arrays of strings, numbers, booleans, quoted strings", "tests": [ { "name": "parses string arrays inline", diff --git a/src/test/resources/conformance/decode/arrays-tabular.json b/src/test/resources/conformance/decode/arrays-tabular.json index 636e2bca..9c18ed9d 100644 --- a/src/test/resources/conformance/decode/arrays-tabular.json +++ b/src/test/resources/conformance/decode/arrays-tabular.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Tabular array decoding - parsing arrays of uniform objects with headers", + "description": "Tabular array decoding – parsing arrays of uniform objects with headers", "tests": [ { "name": "parses tabular arrays of uniform objects", @@ -48,7 +48,7 @@ "specSection": "9.3" }, { - "name": "parses quoted key with tabular array format", + "name": "parses quoted key with tabular array", "input": "\"x-items\"[2]{id,name}:\n 1,Ada\n 2,Bob", "expected": { "x-items": [ @@ -59,7 +59,7 @@ "specSection": "9.3" }, { - "name": "parses quoted empty string key with tabular array format", + "name": "parses quoted empty string key with tabular array", "input": "\"\"[2]{id,name}:\n 1,Ada\n 2,Bob", "expected": { "": [ @@ -70,11 +70,11 @@ "specSection": "9.3" }, { - "name": "treats unquoted colon as terminator for tabular rows and start of key-value pair", - "input": "items[2]{id,name}:\n 1,Alice\n 2,Bob\ncount: 2", + "name": "terminates a tabular scope at a dedented key-value line", + "input": "items[2]{id,name}:\n 1,Ada\n 2,Bob\ncount: 2", "expected": { "items": [ - { "id": 1, "name": "Alice" }, + { "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" } ], "count": 2 @@ -93,6 +93,130 @@ }, "specSection": "9.3", "note": "The dedent to the header's depth closes the rows (§8); the line is a key-value pair, and the whole post-colon token is the value, non-numeric so a string (§11.2)" + }, + { + "name": "parses nested field groups into nested objects", + "input": "orders[2]{id,customer{name,country},total}:\n 1,Ada,DK,99\n 2,Bob,UK,149", + "expected": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, + { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + ] + }, + "specSection": "9.3" + }, + { + "name": "parses sibling nested field groups by depth-first cell assignment", + "input": "shipments[1]{id,sender{name,city},receiver{name,city}}:\n s1,ACME,Berlin,Globex,Oslo", + "expected": { + "shipments": [ + { "id": "s1", "sender": { "name": "ACME", "city": "Berlin" }, "receiver": { "name": "Globex", "city": "Oslo" } } + ] + }, + "specSection": "9.3" + }, + { + "name": "parses nested field groups recursively without a depth cap", + "input": "items[2]{id,geo{point{lat,lon}}}:\n 1,1.5,2.5\n 2,3,4", + "expected": { + "items": [ + { "id": 1, "geo": { "point": { "lat": 1.5, "lon": 2.5 } } }, + { "id": 2, "geo": { "point": { "lat": 3, "lon": 4 } } } + ] + }, + "specSection": "9.3" + }, + { + "name": "parses nested field groups with the pipe delimiter", + "input": "orders[2|]{id|customer{name|country}|total}:\n 1|Ada|DK|99\n 2|Bob|UK|149", + "expected": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, + { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + ] + }, + "specSection": "9.3" + }, + { + "name": "matches braces outside quoted names only when parsing field entries", + "input": "items[1]{\"a{b}\",c}:\n 1,2", + "expected": { + "items": [ + { "a{b}": 1, "c": 2 } + ] + }, + "specSection": "6", + "note": "A quoted field name may contain braces; brace matching is quote-aware" + }, + { + "name": "parses quoted subfield names inside nested field groups", + "input": "items[1]{id,customer{\"full name\",country}}:\n 1,Ada,UK", + "expected": { + "items": [ + { "id": 1, "customer": { "full name": "Ada", "country": "UK" } } + ] + }, + "specSection": "9.3" + }, + { + "name": "applies LWW for duplicate field names in non-strict mode", + "input": "items[1]{a,a}:\n 1,2", + "expected": { + "items": [ + { "a": 2 } + ] + }, + "options": { + "strict": false + }, + "specSection": "14.3", + "note": "Duplicate field names yield duplicate sibling keys per row; non-strict resolves last-write-wins" + }, + { + "name": "applies LWW when a bare field and a nested group share a name in non-strict mode", + "input": "items[1]{a,a{x}}:\n 1,2", + "expected": { + "items": [ + { "a": { "x": 2 } } + ] + }, + "options": { + "strict": false + }, + "specSection": "14.3" + }, + { + "name": "accepts a field name outside the encoder unquoted-key pattern", + "input": "items[1]{2key}:\n 1", + "expected": { + "items": [ + { + "2key": 1 + } + ] + }, + "specSection": "7.4", + "minSpecVersion": "4.1" + }, + { + "name": "keeps every row when the count mismatches in non-strict mode", + "input": "a[1]{x}:\n 1\n 2", + "expected": { + "a": [ + { + "x": 1 + }, + { + "x": 2 + } + ] + }, + "options": { + "strict": false + }, + "specSection": "14.1", + "note": "A declared [N] never truncates a scope", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/decode/blank-lines.json b/src/test/resources/conformance/decode/blank-lines.json index a4dba63b..b8eb4f93 100644 --- a/src/test/resources/conformance/decode/blank-lines.json +++ b/src/test/resources/conformance/decode/blank-lines.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Blank line handling - strict mode errors on blank lines inside arrays, accepts blank lines outside arrays", + "description": "Blank line handling – strict mode errors on blank lines inside arrays, accepts blank lines outside arrays", "tests": [ { "name": "throws on blank line inside list array", @@ -23,6 +23,16 @@ }, "specSection": "14.2" }, + { + "name": "throws on blank line between keyed entry rows", + "input": "m[2:]{v}:\n a: 1\n\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.2" + }, { "name": "throws on multiple blank lines inside array", "input": "items[2]:\n - a\n\n\n - b", @@ -75,8 +85,7 @@ "options": { "strict": true }, - "specSection": "12", - "minSpecVersion": "3.2" + "specSection": "12" }, { "name": "accepts trailing newline at end of file", @@ -139,10 +148,10 @@ }, { "name": "ignores blank lines inside tabular array when strict=false", - "input": "items[2]{id,name}:\n 1,Alice\n\n 2,Bob", + "input": "items[2]{id,name}:\n 1,Ada\n\n 2,Bob", "expected": { "items": [ - { "id": 1, "name": "Alice" }, + { "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" } ] }, @@ -161,6 +170,47 @@ "strict": false }, "specSection": "12" + }, + { + "name": "throws on blank line between list items after nested tabular rows", + "input": "outer[2]:\n - inner[2]{a}:\n 1\n 2\n\n - x", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "The blank is after the inner scope's last row but inside the outer array's span (\u00a712)" + }, + { + "name": "throws on blank line between a list item's fields", + "input": "items[2]:\n - a: 1\n\n b: 2\n - x", + "expected": null, + "shouldError": true, + "specSection": "14.2" + }, + { + "name": "throws on blank line inside the last list item's fields", + "input": "items[1]:\n - a: 1\n\n b: 2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "The header span ends at the last line of the scope's content, not at the last item line (\u00a712)" + }, + { + "name": "accepts blank line between header and first list item", + "input": "items[2]:\n\n - a\n - b", + "expected": { "items": ["a", "b"] }, + "specSection": "12" + }, + { + "name": "accepts blank line between header and first tabular row", + "input": "items[2]{id}:\n\n 1\n 2", + "expected": { "items": [{ "id": 1 }, { "id": 2 }] }, + "specSection": "12" + }, + { + "name": "accepts blank line between header and first entry row", + "input": "m[2:]{v}:\n\n a: 1\n b: 2", + "expected": { "m": { "a": { "v": 1 }, "b": { "v": 2 } } }, + "specSection": "12" } ] } diff --git a/src/test/resources/conformance/decode/comments.json b/src/test/resources/conformance/decode/comments.json new file mode 100644 index 00000000..ac6432d6 --- /dev/null +++ b/src/test/resources/conformance/decode/comments.json @@ -0,0 +1,195 @@ +{ + "version": "4.0", + "category": "decode", + "description": "Comment lines – full-line decode-side comments stripped before structural interpretation", + "tests": [ + { + "name": "strips comment line between tabular rows without terminating rows", + "input": "items[2]{id,name}:\n 1,Ada\n # note\n 2,Bob", + "expected": { + "items": [ + { + "id": 1, + "name": "Ada" + }, + { + "id": 2, + "name": "Bob" + } + ] + }, + "specSection": "5.1", + "note": "The comment is not counted as a row and does not end the tabular scope" + }, + { + "name": "strips comment line between header and first row", + "input": "items[2]{id}:\n # leading\n 1\n 2", + "expected": { + "items": [ + { + "id": 1 + }, + { + "id": 2 + } + ] + }, + "specSection": "5.1" + }, + { + "name": "strips comment line whose indentation is not a multiple of indentSize", + "input": "a: 1\n # three leading spaces\nb: 2", + "expected": { + "a": 1, + "b": 2 + }, + "specSection": "5.1", + "note": "Comment lines are exempt from the §12 strict indentation check" + }, + { + "name": "strips outdented comment inside nested object scope without closing it", + "input": "user:\n id: 1\n# outdented note\n name: Ada", + "expected": { + "user": { + "id": 1, + "name": "Ada" + } + }, + "specSection": "5.1", + "note": "A depth-0 comment between depth-1 fields does not close the object" + }, + { + "name": "strips comment inside list array without counting as item", + "input": "items[2]:\n - a\n # note\n - b", + "expected": { + "items": [ + "a", + "b" + ] + }, + "specSection": "5.1" + }, + { + "name": "strips comment before root array header", + "input": "# heading\n[2]: a,b", + "expected": [ + "a", + "b" + ], + "specSection": "5.1", + "note": "Root-form discovery runs on the comment-stripped sequence" + }, + { + "name": "strips comment containing an unterminated quote without unescaping", + "input": "# \"unterminated\na: 1", + "expected": { + "a": 1 + }, + "specSection": "5.1", + "note": "Comment text is discarded without interpretation; the quote never reaches the tokenizer" + }, + { + "name": "decodes document of only comments to empty object", + "input": "# only\n# comments", + "expected": {}, + "specSection": "5.1" + }, + { + "name": "decodes comments and blank lines only to empty object", + "input": "\n# c\n\n# d\n", + "expected": {}, + "specSection": "5.1" + }, + { + "name": "decodes a hash-leading root line as a comment, yielding an empty object", + "input": "#hello", + "expected": {}, + "specSection": "5.1", + "note": "The only line is a comment, so the document is empty after the pre-pass" + }, + { + "name": "throws when a stripped hash-leading row breaks the declared count", + "input": "items[2]{id}:\n #1\n 2", + "expected": null, + "shouldError": true, + "specSection": "14.1", + "note": "The first cell makes the whole line a comment; only one row remains for [2]" + }, + { + "name": "drops hash-leading row silently in non-strict mode", + "input": "items[2]{id}:\n #1\n 2", + "expected": { + "items": [ + { + "id": 2 + } + ] + }, + "options": { + "strict": false + }, + "specSection": "5.1", + "note": "A hash-leading first cell makes the whole line a comment; non-strict mode drops the row silently" + }, + { + "name": "throws on tab-indented hash line, which is not a comment", + "input": "a: 1\n\t# not a comment", + "expected": null, + "shouldError": true, + "specSection": "5.1", + "note": "Only spaces (U+0020) may precede the #; the tab makes this a regular line, and tabs in indentation error in strict mode (§12)" + }, + { + "name": "parses quoted hash-leading first cell as data, not comment", + "input": "items[1]{tag}:\n \"#a\"", + "expected": { + "items": [ + { + "tag": "#a" + } + ] + }, + "specSection": "5.1" + }, + { + "name": "parses hyphen list item with hash-leading token as string", + "input": "items[1]:\n - #x", + "expected": { + "items": [ + "#x" + ] + }, + "specSection": "5.1", + "note": "\"#\" is not the line's first non-space character; comments are full-line only" + }, + { + "name": "parses hash after key-value colon as string value", + "input": "note: #x", + "expected": { + "note": "#x" + }, + "specSection": "5.1", + "note": "No inline comments; the unquoted token decodes as the string #x" + }, + { + "name": "parses hash mid-value as data, not a trailing comment", + "input": "msg: hello # world", + "expected": { + "msg": "hello # world" + }, + "specSection": "5.1" + }, + { + "name": "round-trips quoted hash-leading values in field and inline array positions", + "input": "tag: \"#x\"\nlist[2]: \"#a\",b", + "expected": { + "tag": "#x", + "list": [ + "#a", + "b" + ] + }, + "specSection": "7.2" + } + ] +} diff --git a/src/test/resources/conformance/decode/delimiters.json b/src/test/resources/conformance/decode/delimiters.json index 021505bc..847f87b2 100644 --- a/src/test/resources/conformance/decode/delimiters.json +++ b/src/test/resources/conformance/decode/delimiters.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Delimiter decoding - tab and pipe delimiter parsing, delimiter-aware value splitting", + "description": "Delimiter decoding – tab and pipe delimiter parsing, delimiter-aware value splitting", "tests": [ { "name": "parses primitive arrays with tab delimiter", @@ -63,8 +63,7 @@ "expected": { "items": [{ "tags": ["a", "b", "c"] }] }, - "specSection": "11", - "note": "Parent uses tab, nested defaults to comma" + "specSection": "11" }, { "name": "parses nested arrays inside list items with default comma delimiter when parent uses pipe", @@ -162,13 +161,13 @@ "note": "Object field values are parsed as the whole post-colon token, never split on a delimiter (§11.2)" }, { - "name": "object values in list items follow document delimiter", + "name": "does not split object values on commas inside a tab-delimited scope", "input": "items[2\t]:\n - status: a,b\n - status: c,d", "expected": { "items": [{ "status": "a,b" }, { "status": "c,d" }] }, - "specSection": "11", - "note": "Active delimiter is tab, but object values use document delimiter for quoting" + "specSection": "11.2", + "note": "The active delimiter splits row and inline cells, never object field values" }, { "name": "parses quoted comma in object values", @@ -233,6 +232,18 @@ "items": [{ "a|b": 1 }, { "a|b": 2 }] }, "specSection": "11" + }, + { + "name": "falls through to a key-value line on a header delimiter mismatch", + "input": "a[2|]{x,y}: 1|2", + "expected": { + "a[2|]{x,y}": "1|2" + }, + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/decode/indentation-errors.json b/src/test/resources/conformance/decode/indentation-errors.json index 728650ec..96097d00 100644 --- a/src/test/resources/conformance/decode/indentation-errors.json +++ b/src/test/resources/conformance/decode/indentation-errors.json @@ -1,43 +1,43 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Strict mode indentation validation - non-multiple indentation, tab characters, custom indent sizes", + "description": "Strict mode indentation validation – non-multiple indentation, tab characters, custom indentSizes", "tests": [ { - "name": "throws on object field with non-multiple indentation (3 spaces with indent=2)", + "name": "throws on object field with non-multiple indentation (3 spaces with indentSize 2)", "input": "a:\n b: 1", "expected": null, "shouldError": true, "options": { - "indent": 2, + "indentSize": 2, "strict": true }, "specSection": "14.2" }, { - "name": "throws on list item with non-multiple indentation (3 spaces with indent=2)", + "name": "throws on list item with non-multiple indentation (3 spaces with indentSize 2)", "input": "items[2]:\n - id: 1\n - id: 2", "expected": null, "shouldError": true, "options": { - "indent": 2, + "indentSize": 2, "strict": true }, "specSection": "14.2" }, { - "name": "throws on non-multiple indentation with custom indent=4 (3 spaces)", + "name": "throws on non-multiple indentation with custom indentSize 4 (3 spaces)", "input": "a:\n b: 1", "expected": null, "shouldError": true, "options": { - "indent": 4, + "indentSize": 4, "strict": true }, "specSection": "14.2" }, { - "name": "accepts correct indentation with custom indent size (4 spaces with indent=4)", + "name": "accepts correct indentation with custom indentSize (4 spaces with indentSize 4)", "input": "a:\n b: 1", "expected": { "a": { @@ -45,7 +45,7 @@ } }, "options": { - "indent": 4, + "indentSize": 4, "strict": true }, "specSection": "12" @@ -122,7 +122,7 @@ } }, "options": { - "indent": 2, + "indentSize": 2, "strict": false }, "specSection": "12" @@ -138,10 +138,64 @@ } }, "options": { - "indent": 2, + "indentSize": 2, "strict": false }, "specSection": "12" + }, + { + "name": "throws on depth jump of more than one level", + "input": "a:\n b: 1", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "Four leading spaces are a multiple of indentSize 2, but depth 2 directly under depth 0 skips a level" + }, + { + "name": "throws on depth jump inside a nested object", + "input": "a:\n b:\n c: 1", + "expected": null, + "shouldError": true, + "specSection": "14.2" + }, + { + "name": "throws on over-indented line after a primitive field", + "input": "a: 1\n b: 2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "A primitive field opens no scope, so the depth-2 line belongs to none and must not be silently discarded" + }, + { + "name": "throws on a line indented one level under a primitive field", + "input": "a: 1\n b: 2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "Even a single extra level is over-indentation when the preceding line did not open a scope" + }, + { + "name": "throws on over-indented line inside a nested object", + "input": "a:\n b: 1\n c: 2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "The depth-3 line follows a primitive field at depth 1; it belongs to no scope" + }, + { + "name": "throws on over-indented line after tabular rows", + "input": "u[1]{a}:\n 1\n junk: 9", + "expected": null, + "shouldError": true, + "specSection": "14.2" + }, + { + "name": "throws on orphan scalar line under a primitive field", + "input": "a: 1\n hello", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "A scalar line is valid only at root primitive position" } ] } diff --git a/src/test/resources/conformance/decode/numbers.json b/src/test/resources/conformance/decode/numbers.json index d15cdac4..22ac3f15 100644 --- a/src/test/resources/conformance/decode/numbers.json +++ b/src/test/resources/conformance/decode/numbers.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Number decoding edge cases - trailing zeros, exponent forms, negative zero", + "description": "Number decoding edge cases – trailing zeros, exponent forms, negative zero", "tests": [ { "name": "parses number with trailing zeros in fractional part", @@ -9,8 +9,7 @@ "expected": { "value": 1.5 }, - "specSection": "4", - "note": "Decoders accept trailing zeros; numeric value is 1.5" + "specSection": "4" }, { "name": "parses negative number with positive exponent", @@ -18,8 +17,7 @@ "expected": { "value": -1000 }, - "specSection": "4", - "note": "Exponent forms are accepted by decoders" + "specSection": "4" }, { "name": "parses lowercase exponent", @@ -69,8 +67,7 @@ "expected": { "value": "05" }, - "specSection": "4", - "note": "Forbidden leading zeros cause tokens to be treated as strings" + "specSection": "4" }, { "name": "parses very small exponent", @@ -94,8 +91,7 @@ "expected": { "value": 0 }, - "specSection": "4", - "note": "Exponent forms with a zero integer part (0e1) are valid numbers" + "specSection": "4" }, { "name": "parses negative zero with exponent as number", @@ -103,8 +99,7 @@ "expected": { "value": 0 }, - "specSection": "4", - "note": "Negative zero with exponent (-0e1) decodes to numeric 0" + "specSection": "4" }, { "name": "parses exponent notation", @@ -128,8 +123,7 @@ "name": "treats unquoted leading-zero number as string", "input": "05", "expected": "05", - "specSection": "4", - "note": "Leading zeros make it a string" + "specSection": "4" }, { "name": "treats unquoted multi-leading-zero as string", @@ -143,12 +137,6 @@ "expected": "0123", "specSection": "4" }, - { - "name": "treats leading-zero in object value as string", - "input": "a: 05", - "expected": { "a": "05" }, - "specSection": "4" - }, { "name": "treats leading-zeros in array as strings", "input": "nums[3]: 05,007,0123", @@ -159,8 +147,7 @@ "name": "treats unquoted negative leading-zero number as string", "input": "-05", "expected": "-05", - "specSection": "4", - "note": "Negative numbers with leading zeros in the integer part are treated as strings" + "specSection": "4" }, { "name": "treats negative leading-zeros in array as strings", @@ -169,6 +156,70 @@ "nums": ["-05", "-007"] }, "specSection": "4" + }, + { + "name": "treats bare fraction without integer part as string", + "input": "value: .5", + "expected": { + "value": ".5" + }, + "specSection": "4", + "note": "Outside the number grammar, so a host parser would read 0.5 and corrupt the round-trip" + }, + { + "name": "treats integer with trailing dot as string", + "input": "value: 1.", + "expected": { + "value": "1." + }, + "specSection": "4" + }, + { + "name": "treats leading-plus integer as string", + "input": "value: +1", + "expected": { + "value": "+1" + }, + "specSection": "4", + "note": "The grammar admits only an optional leading minus, so +1 stays a string" + }, + { + "name": "treats leading-plus tokens in array as strings", + "input": "values[3]: +1,+1.5,+1e2", + "expected": { + "values": ["+1", "+1.5", "+1e2"] + }, + "specSection": "4", + "note": "The grammar admits only an optional leading minus, so leading-plus cells stay strings" + }, + { + "name": "parses positive exponent sign as number", + "input": "value: 1e+2", + "expected": { + "value": 100 + }, + "specSection": "4", + "note": "The plus sign is inside the exponent, which the grammar allows" + }, + { + "name": "treats Infinity and NaN tokens as strings", + "input": "a: Infinity\nb: -Infinity\nc: NaN", + "expected": { + "a": "Infinity", + "b": "-Infinity", + "c": "NaN" + }, + "specSection": "4" + }, + { + "name": "treats hex and underscore-separated tokens as strings", + "input": "a: 0x10\nb: 1_000", + "expected": { + "a": "0x10", + "b": "1_000" + }, + "specSection": "4", + "note": "Host parsers that accept these forms have a wider grammar than the spec permits" } ] } diff --git a/src/test/resources/conformance/decode/objects-keyed.json b/src/test/resources/conformance/decode/objects-keyed.json new file mode 100644 index 00000000..3e2e05de --- /dev/null +++ b/src/test/resources/conformance/decode/objects-keyed.json @@ -0,0 +1,230 @@ +{ + "version": "4.0", + "category": "decode", + "description": "Keyed tabular decoding – parsing keyed headers and entry rows into objects", + "tests": [ + { + "name": "parses keyed tabular objects", + "input": "servers[2:]{host,port}:\n alpha: a.example.com,8080\n beta: b.example.com,9090", + "expected": { + "servers": { + "alpha": { "host": "a.example.com", "port": 8080 }, + "beta": { "host": "b.example.com", "port": 9090 } + } + }, + "specSection": "9.5" + }, + { + "name": "parses a keyless keyed header as a root object", + "input": "[2:]{age,city}:\n alice: 30,Berlin\n bob: 25,Oslo", + "expected": { + "alice": { "age": 30, "city": "Berlin" }, + "bob": { "age": 25, "city": "Oslo" } + }, + "specSection": "9.5", + "note": "Root-form discovery (§5) routes the keyless keyed header to a root object, not a root array" + }, + { + "name": "parses nested field groups in keyed headers", + "input": "regions[2:]{name,geo{lat,lon}}:\n eu: Europe,50,10\n us: America,40,-100", + "expected": { + "regions": { + "eu": { "name": "Europe", "geo": { "lat": 50, "lon": 10 } }, + "us": { "name": "America", "geo": { "lat": 40, "lon": -100 } } + } + }, + "specSection": "9.5" + }, + { + "name": "parses keyed headers with the pipe delimiter", + "input": "servers[2:|]{host|port}:\n alpha: a|8080\n beta: b|9090", + "expected": { + "servers": { + "alpha": { "host": "a", "port": 8080 }, + "beta": { "host": "b", "port": 9090 } + } + }, + "specSection": "9.5" + }, + { + "name": "parses keyed headers with the tab delimiter", + "input": "servers[2:\t]{host\tport}:\n alpha: a\t8080\n beta: b\t9090", + "expected": { + "servers": { + "alpha": { "host": "a", "port": 8080 }, + "beta": { "host": "b", "port": 9090 } + } + }, + "specSection": "9.5" + }, + { + "name": "parses quoted entry keys", + "input": "ids[2:]{v}:\n \"42\": 1\n \"my key\": 2", + "expected": { + "ids": { + "42": { "v": 1 }, + "my key": { "v": 2 } + } + }, + "specSection": "9.5" + }, + { + "name": "parses a quoted entry key containing a colon", + "input": "m[1:]{v}:\n \"a:b\": 1", + "expected": { + "m": { + "a:b": { "v": 1 } + } + }, + "specSection": "9.5", + "note": "The entry-row split is at the first unquoted colon; the quoted colon is key data" + }, + { + "name": "treats a key-value line at the header's depth as a sibling after entries end", + "input": "data[2:]{x}:\n a: 1\n b: 2\ncount: 3", + "expected": { + "data": { + "a": { "x": 1 }, + "b": { "x": 2 } + }, + "count": 3 + }, + "specSection": "9.5", + "note": "A keyed scope ends only by dedent or end of input" + }, + { + "name": "treats key-value-shaped lines at entry depth as entry rows", + "input": "m[2:]{x}:\n total: 5\n other: 6", + "expected": { + "m": { + "total": { "x": 5 }, + "other": { "x": 6 } + } + }, + "specSection": "9.5", + "note": "Opposite polarity to §9.3: in a tabular array, a colon-before-delimiter line at row depth ends the rows; in a keyed scope it is an entry" + }, + { + "name": "splits an entry row at its first unquoted colon before delimiter splitting", + "input": "m[1:]{v}:\n k[2]: 5", + "expected": { + "m": { + "k[2]": { "v": 5 } + } + }, + "specSection": "9.5", + "note": "The token before the first unquoted colon is the entry key, accepted as a literal key per §7.4 even though an encoder would quote it" + }, + { + "name": "decodes the cell token [] as a string, not an empty array", + "input": "m[2:]{v}:\n a: []\n b: x", + "expected": { + "m": { + "a": { "v": "[]" }, + "b": { "v": "x" } + } + }, + "specSection": "9.5", + "note": "The §9.1 empty-array form does not apply inside entry rows" + }, + { + "name": "accepts a declared entry count of zero", + "input": "m[0:]{v}:", + "expected": { + "m": {} + }, + "specSection": "9.5", + "note": "Encoders never emit keyed headers for fewer than two entries; decoders accept any N with count checks" + }, + { + "name": "accepts a single entry row", + "input": "m[1:]{v}:\n a: 1", + "expected": { + "m": { + "a": { "v": 1 } + } + }, + "specSection": "9.5" + }, + { + "name": "parses quoted cells containing the active delimiter", + "input": "m[2:]{t}:\n a: \"x,y\"\n b: z", + "expected": { + "m": { + "a": { "t": "x,y" }, + "b": { "t": "z" } + } + }, + "specSection": "9.5" + }, + { + "name": "parses a keyed header on a hyphen line", + "input": "items[2]:\n - config[2:]{x}:\n a: 1\n b: 2\n status: ok\n - status: down", + "expected": { + "items": [ + { "config": { "a": { "x": 1 }, "b": { "x": 2 } }, "status": "ok" }, + { "status": "down" } + ] + }, + "specSection": "10" + }, + { + "name": "ignores comment lines between entry rows", + "input": "m[2:]{v}:\n a: 1\n# note\n b: 2", + "expected": { + "m": { + "a": { "v": 1 }, + "b": { "v": 2 } + } + }, + "specSection": "5.1", + "note": "Comment removal never ends a keyed scope, and a comment line never counts as an entry" + }, + { + "name": "applies LWW for duplicate entry keys in non-strict mode", + "input": "m[2:]{v}:\n a: 1\n a: 2", + "expected": { + "m": { + "a": { "v": 2 } + } + }, + "options": { + "strict": false + }, + "specSection": "14.3", + "note": "Entry keys are sibling keys of the decoded object and resolve last-write-wins" + }, + { + "name": "accepts entry keys outside the encoder unquoted-key pattern", + "input": "u[2:]{a}:\n 2key: 1\n foo-bar: 2", + "expected": { + "u": { + "2key": { + "a": 1 + }, + "foo-bar": { + "a": 2 + } + } + }, + "specSection": "7.4", + "minSpecVersion": "4.1" + }, + { + "name": "skips an entry-depth line without a colon in non-strict mode", + "input": "u[2:]{x}:\n a: 1\n boom", + "expected": { + "u": { + "a": { + "x": 1 + } + } + }, + "options": { + "strict": false + }, + "specSection": "9.5", + "minSpecVersion": "4.1" + } + ] +} diff --git a/src/test/resources/conformance/decode/objects.json b/src/test/resources/conformance/decode/objects.json index 68b249ed..6987da74 100644 --- a/src/test/resources/conformance/decode/objects.json +++ b/src/test/resources/conformance/decode/objects.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "decode", - "description": "Object decoding - fields, nested objects, key parsing, §6 fall-through (non-strict), and §14.4 duplicate-key LWW", + "description": "Object decoding – fields, nested objects, key parsing, §6 fall-through (non-strict), and §14.3 duplicate-key LWW", "tests": [ { "name": "parses objects with primitive values", @@ -47,8 +47,7 @@ "options": { "strict": false }, - "specSection": "14.4", - "minSpecVersion": "3.2" + "specSection": "14.3" }, { "name": "parses quoted object value with colon", @@ -72,7 +71,7 @@ "expected": { "text": "line1\nline2" }, - "specSection": "8" + "specSection": "7.1" }, { "name": "parses quoted object value with escaped quotes", @@ -80,7 +79,7 @@ "expected": { "text": "say \"hello\"" }, - "specSection": "8" + "specSection": "7.1" }, { "name": "parses quoted object value with leading/trailing spaces", @@ -210,6 +209,18 @@ "specSection": "6", "note": "Non-whitespace [bar] between ] and : prevents array header interpretation; non-strict fall-through produces a literal key not constrained by §7.3" }, + { + "name": "treats bracket segment without a length as literal key (non-strict)", + "input": "key[]: 1,2", + "options": { + "strict": false + }, + "expected": { + "key[]": "1,2" + }, + "specSection": "6", + "note": "An absent length prevents header interpretation; non-strict fall-through produces a literal key" + }, { "name": "treats non-integer bracket content as literal key (non-strict)", "input": "foo[bar][1]: 20", @@ -320,7 +331,7 @@ "expected": { "line\nbreak": 1 }, - "specSection": "8" + "specSection": "7.1" }, { "name": "unescapes tab in key", @@ -328,7 +339,15 @@ "expected": { "tab\there": 2 }, - "specSection": "8" + "specSection": "7.1" + }, + { + "name": "unescapes tab in key immediately followed by a colon", + "input": "\"\\t:x\": v", + "expected": { + "\t:x": "v" + }, + "specSection": "7.1" }, { "name": "unescapes tab in key immediately followed by a colon", @@ -344,7 +363,7 @@ "expected": { "he said \"hi\"": 1 }, - "specSection": "8" + "specSection": "7.1" }, { "name": "parses deeply nested objects with indentation", @@ -369,8 +388,7 @@ "options": { "strict": false }, - "specSection": "14.4", - "minSpecVersion": "3.2" + "specSection": "14.3" }, { "name": "applies LWW for duplicate keys within a list-item object in non-strict mode", @@ -385,8 +403,120 @@ "options": { "strict": false }, - "specSection": "14.4", - "minSpecVersion": "3.2" + "specSection": "14.3" + }, + { + "name": "accepts unquoted key outside the encoder key pattern (hyphen)", + "input": "foo-bar: 1", + "expected": { + "foo-bar": 1 + }, + "specSection": "7.4", + "note": "§7.3 constrains encoder output; decoders accept any token before the first unquoted colon as a literal key" + }, + { + "name": "accepts unquoted key outside the encoder key pattern (leading digit)", + "input": "2key: x", + "expected": { + "2key": "x" + }, + "specSection": "7.4" + }, + { + "name": "accepts unquoted hyphen-leading value in strict mode", + "input": "k: -x", + "expected": { + "k": "-x" + }, + "specSection": "7.4", + "note": "Encoders must quote hyphen-leading strings (§7.2); decoders accept the unquoted form as data" + }, + { + "name": "accepts unquoted colon-containing value in strict mode", + "input": "k: b:c", + "expected": { + "k": "b:c" + }, + "specSection": "7.4", + "note": "The first unquoted colon ends the key; the entire post-colon token is the value (§11.2)" + }, + { + "name": "materializes __proto__ as an ordinary own key", + "input": "__proto__: polluted", + "expected": { + "__proto__": "polluted" + }, + "specSection": "15", + "note": "Decoding MUST NOT mutate the host object model, so the decoded object carries __proto__ as an own entry" + }, + { + "name": "materializes constructor and prototype as ordinary own keys", + "input": "constructor: 1\nprototype: 2", + "expected": { + "constructor": 1, + "prototype": 2 + }, + "specSection": "15" + }, + { + "name": "materializes __proto__ opening a nested object as an ordinary own key", + "input": "__proto__:\n admin: true", + "expected": { + "__proto__": { + "admin": true + } + }, + "specSection": "15", + "note": "The classic pollution shape – the nested object becomes an own entry, never the prototype" + }, + { + "name": "materializes quoted __proto__ key as an ordinary own key", + "input": "\"__proto__\": x", + "expected": { + "__proto__": "x" + }, + "specSection": "15" + }, + { + "name": "materializes __proto__ tabular field name as ordinary own keys", + "input": "rows[2]{__proto__,x}:\n a,1\n b,2", + "expected": { + "rows": [ + { + "__proto__": "a", + "x": 1 + }, + { + "__proto__": "b", + "x": 2 + } + ] + }, + "specSection": "15" + }, + { + "name": "falls through to a key-value line when whitespace precedes the bracket segment", + "input": "foo [2]: bar,baz", + "expected": { + "foo [2]": "bar,baz" + }, + "options": { + "strict": false + }, + "specSection": "5.2", + "minSpecVersion": "4.1" + }, + { + "name": "accepts a header key outside the encoder unquoted-key pattern", + "input": "foo-bar[2]: 1,2", + "expected": { + "foo-bar": [ + 1, + 2 + ] + }, + "specSection": "7.4", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/decode/path-expansion.json b/src/test/resources/conformance/decode/path-expansion.json deleted file mode 100644 index 0b513c67..00000000 --- a/src/test/resources/conformance/decode/path-expansion.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "version": "1.5", - "category": "decode", - "description": "Path expansion with safe mode, deep merge, conflict resolution tied to strict mode", - "tests": [ - { - "name": "expands dotted key to nested object in safe mode", - "input": "a.b.c: 1", - "expected": { - "a": { - "b": { - "c": 1 - } - } - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4" - }, - { - "name": "expands dotted key with inline array", - "input": "data.meta.items[2]: a,b", - "expected": { - "data": { - "meta": { - "items": ["a", "b"] - } - } - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4" - }, - { - "name": "expands dotted key with tabular array", - "input": "a.b.items[2]{id,name}:\n 1,A\n 2,B", - "expected": { - "a": { - "b": { - "items": [ - { "id": 1, "name": "A" }, - { "id": 2, "name": "B" } - ] - } - } - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4" - }, - { - "name": "preserves literal dotted keys when expansion is off", - "input": "user.name: Ada", - "expected": { - "user.name": "Ada" - }, - "options": { - "expandPaths": "off" - }, - "specSection": "13.4" - }, - { - "name": "expands and deep-merges preserving document-order insertion", - "input": "a.b.c: 1\na.b.d: 2\na.e: 3", - "expected": { - "a": { - "b": { - "c": 1, - "d": 2 - }, - "e": 3 - } - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4" - }, - { - "name": "throws on expansion conflict (object vs primitive) when strict=true", - "input": "a.b: 1\na: 2", - "expected": null, - "shouldError": true, - "options": { - "expandPaths": "safe", - "strict": true - }, - "specSection": "14.3" - }, - { - "name": "throws on expansion conflict (object vs array) when strict=true", - "input": "a.b: 1\na[2]: 2,3", - "expected": null, - "shouldError": true, - "options": { - "expandPaths": "safe", - "strict": true - }, - "specSection": "14.3" - }, - { - "name": "applies LWW when strict=false (primitive overwrites expanded object)", - "input": "a.b: 1\na: 2", - "expected": { - "a": 2 - }, - "options": { - "expandPaths": "safe", - "strict": false - }, - "specSection": "13.4", - "note": "Document order determines winner: later key overwrites earlier" - }, - { - "name": "applies LWW when strict=false (expanded object overwrites primitive)", - "input": "a: 1\na.b: 2", - "expected": { - "a": { - "b": 2 - } - }, - "options": { - "expandPaths": "safe", - "strict": false - }, - "specSection": "13.4", - "note": "Document order determines winner: later key overwrites earlier" - }, - { - "name": "preserves quoted dotted key as literal when expandPaths=safe", - "input": "a.b: 1\n\"c.d\": 2", - "expected": { - "a": { - "b": 1 - }, - "c.d": 2 - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4" - }, - { - "name": "preserves quoted non-IdentifierSegment keys as literals", - "input": "\"full-name.x\": 1", - "expected": { - "full-name.x": 1 - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4", - "note": "Quoted keys remain literal after unescaping; safe-mode expansion does not split them. The key must be quoted because §7.3 forbids hyphens in unquoted keys." - }, - { - "name": "expands keys creating empty nested objects", - "input": "a.b.c:", - "expected": { - "a": { - "b": { - "c": {} - } - } - }, - "options": { - "expandPaths": "safe" - }, - "specSection": "13.4" - } - ] -} diff --git a/src/test/resources/conformance/decode/primitives.json b/src/test/resources/conformance/decode/primitives.json index 4efb6016..23479300 100644 --- a/src/test/resources/conformance/decode/primitives.json +++ b/src/test/resources/conformance/decode/primitives.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "decode", - "description": "Primitive value decoding - strings, numbers, booleans, null, unescaping", + "description": "Primitive value decoding – strings, numbers, booleans, null, unescaping", "tests": [ { "name": "parses safe unquoted string", @@ -127,8 +127,7 @@ "name": "respects ambiguity quoting for true", "input": "\"true\"", "expected": "true", - "specSection": "7.4", - "note": "Quoted primitive remains string" + "specSection": "7.4" }, { "name": "respects ambiguity quoting for false", diff --git a/src/test/resources/conformance/decode/root-form.json b/src/test/resources/conformance/decode/root-form.json index 1da36e52..32bc71f7 100644 --- a/src/test/resources/conformance/decode/root-form.json +++ b/src/test/resources/conformance/decode/root-form.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Root form detection - empty document, single primitive, literal empty array", + "description": "Root form detection – empty document, single primitive, literal empty array", "tests": [ { "name": "parses empty document as empty object", @@ -10,8 +10,7 @@ "options": { "strict": true }, - "specSection": "5", - "note": "Empty input (no non-empty lines) decodes to empty object" + "specSection": "5" }, { "name": "parses single primitive string at root as primitive", @@ -48,6 +47,37 @@ "strict": true }, "specSection": "5" + }, + { + "name": "throws on trailing content after a root array", + "input": "[2]: 1,2\njunk: 3", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "5", + "note": "The root form spans the whole document; trailing lines must not be silently discarded" + }, + { + "name": "throws on trailing content after a keyed tabular root", + "input": "[2:]{v}:\n a: 1\n b: 2\njunk: 3", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "5" + }, + { + "name": "throws on trailing content after a root empty array", + "input": "[]\njunk: 3", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "5" } ] } diff --git a/src/test/resources/conformance/decode/validation-errors.json b/src/test/resources/conformance/decode/validation-errors.json index a50b28bb..a174a9ce 100644 --- a/src/test/resources/conformance/decode/validation-errors.json +++ b/src/test/resources/conformance/decode/validation-errors.json @@ -1,17 +1,17 @@ { - "version": "3.1", + "version": "4.0", "category": "decode", - "description": "Validation errors - length mismatches, invalid escapes, syntax errors, delimiter mismatches", + "description": "Validation errors – length mismatches, invalid escapes, syntax errors, delimiter mismatches", "tests": [ { - "name": "throws on array length mismatch (inline primitives - too many)", + "name": "throws when inline values outnumber the declared length", "input": "tags[2]: a,b,c", "expected": null, "shouldError": true, "specSection": "14.1" }, { - "name": "throws on array length mismatch (list format - too many)", + "name": "throws when list items outnumber the declared length", "input": "items[1]:\n - 1\n - 2", "expected": null, "shouldError": true, @@ -39,14 +39,14 @@ "specSection": "14.2" }, { - "name": "rejects truncated unicode escape \\u00b", + "name": "throws on truncated unicode escape \\u00b", "input": "val: \"a\\u00b\"", "expected": null, "shouldError": true, "specSection": "7.1" }, { - "name": "rejects lone surrogate code point \\uD800", + "name": "throws on lone surrogate code point \\uD800", "input": "val: \"a\\uD800b\"", "expected": null, "shouldError": true, @@ -92,8 +92,7 @@ "options": { "strict": true }, - "specSection": "6", - "minSpecVersion": "3.2" + "specSection": "6" }, { "name": "throws on extra brackets between bracket segment and colon in strict mode", @@ -104,8 +103,7 @@ "strict": true }, "specSection": "6", - "note": "Non-whitespace content between ] and : must error in strict mode (§6 fall-through is non-strict only)", - "minSpecVersion": "3.2" + "note": "Non-whitespace content between ] and : must error in strict mode; the fall-through is non-strict only" }, { "name": "throws on text between bracket segment and colon in strict mode", @@ -115,8 +113,7 @@ "options": { "strict": true }, - "specSection": "6", - "minSpecVersion": "3.2" + "specSection": "6" }, { "name": "throws on non-integer bracket segment in strict mode", @@ -126,8 +123,7 @@ "options": { "strict": true }, - "specSection": "6", - "minSpecVersion": "3.2" + "specSection": "6" }, { "name": "throws on duplicate sibling keys in strict mode", @@ -137,8 +133,7 @@ "options": { "strict": true }, - "specSection": "14.4", - "minSpecVersion": "3.2" + "specSection": "14.3" }, { "name": "throws on array header missing colon", @@ -148,14 +143,14 @@ "specSection": "6" }, { - "name": "throws on inline primitive array length mismatch (too few)", + "name": "throws when inline values fall short of the declared length", "input": "tags[3]: a,b", "expected": null, "shouldError": true, "specSection": "14.1" }, { - "name": "throws on list items length mismatch (too few)", + "name": "throws when list items fall short of the declared length", "input": "items[2]:\n - a", "expected": null, "shouldError": true, @@ -170,8 +165,7 @@ "strict": true }, "specSection": "6", - "note": "[03] is not a canonical non-negative integer length; decoders MUST NOT interpret it as a bracket segment", - "minSpecVersion": "3.2" + "note": "[03] is not a canonical non-negative integer length; decoders MUST NOT interpret it as a bracket segment" }, { "name": "throws on negative bracket length in strict mode", @@ -182,8 +176,40 @@ "strict": true }, "specSection": "6", - "note": "[-1] is not a non-negative integer length; decoders MUST NOT interpret it as a bracket segment", - "minSpecVersion": "3.2" + "note": "[-1] is not a non-negative integer length; decoders MUST NOT interpret it as a bracket segment" + }, + { + "name": "throws on decimal bracket length in strict mode", + "input": "x[3.7]: a,b,c", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6", + "note": "[3.7] is not an integer length – lengths are DIGIT-only – so decoders MUST NOT interpret it as a bracket segment" + }, + { + "name": "throws on bracket length with plus sign in strict mode", + "input": "x[+3]: a,b,c", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6", + "note": "[+3] carries a sign and is not a non-negative integer length; decoders MUST NOT interpret it as a bracket segment" + }, + { + "name": "throws on bracket length in exponent form in strict mode", + "input": "x[1e1]: 1,2,3,4,5,6,7,8,9,10", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6", + "note": "[1e1] is exponent notation, not a DIGIT-only integer length; decoders MUST NOT interpret it as a bracket segment" }, { "name": "throws on decimal bracket length in strict mode", @@ -230,11 +256,10 @@ "strict": true }, "specSection": "6", - "note": "No whitespace is permitted between ] and the colon/fields segment; any content there prevents header interpretation", - "minSpecVersion": "3.2" + "note": "No whitespace is permitted between ] and the colon/field list; any content there prevents header interpretation" }, { - "name": "throws on whitespace between bracket segment and fields segment in strict mode", + "name": "throws on whitespace between bracket segment and field list in strict mode", "input": "items[2] {a,b}:\n 1,2\n 3,4", "expected": null, "shouldError": true, @@ -242,8 +267,7 @@ "strict": true }, "specSection": "6", - "note": "No whitespace is permitted between ] and the fields segment; mirrors the ]-to-colon rule", - "minSpecVersion": "3.2" + "note": "No whitespace is permitted between ] and the field list; mirrors the ]-to-colon rule" }, { "name": "throws on nested duplicate sibling keys in strict mode", @@ -253,8 +277,7 @@ "options": { "strict": true }, - "specSection": "14.4", - "minSpecVersion": "3.2" + "specSection": "14.3" }, { "name": "throws on duplicate keys within a list-item object in strict mode", @@ -264,8 +287,258 @@ "options": { "strict": true }, - "specSection": "14.4", - "minSpecVersion": "3.2" + "specSection": "14.3" + }, + { + "name": "throws on bracket segment without a length", + "input": "key[]: 1,2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "An absent length is a malformed bracket segment, same class as [03] and [-1]; only key: [] (bracket after the colon) means an empty array" + }, + { + "name": "throws on row cell count not matching the leaf-field count", + "input": "orders[1]{id,customer{name,country}}:\n 1,Ada", + "expected": null, + "shouldError": true, + "specSection": "14.1", + "note": "The header declares three leaf fields (id, customer.name, customer.country); the row has two cells" + }, + { + "name": "throws on empty field list in strict mode", + "input": "items[1]{}:\n 1", + "expected": null, + "shouldError": true, + "specSection": "14.2" + }, + { + "name": "throws on empty nested field group in strict mode", + "input": "items[1]{id,meta{}}:\n 1", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "A field list must contain at least one field entry at every nesting level" + }, + { + "name": "throws on unmatched brace in field list in strict mode", + "input": "items[1]{id,customer{name:\n 1,Ada", + "expected": null, + "shouldError": true, + "specSection": "14.2" + }, + { + "name": "throws on duplicate field names at the same field-list level in strict mode", + "input": "items[1]{a,a{x}}:\n 1,2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.3", + "note": "A bare field and a nested field group sharing one name yield duplicate sibling keys in every decoded row" + }, + { + "name": "throws on entry row count mismatch with keyed header length", + "input": "m[2:]{v}:\n a: 1", + "expected": null, + "shouldError": true, + "specSection": "14.1" + }, + { + "name": "throws on entry row cell count not matching the leaf-field count", + "input": "m[1:]{a,b}:\n k: 1", + "expected": null, + "shouldError": true, + "specSection": "14.1" + }, + { + "name": "throws on an entry row with no cells after the entry key", + "input": "m[1:]{v}:\n a:", + "expected": null, + "shouldError": true, + "specSection": "14.1", + "note": "A bare entrykey: has zero cells, and a field list always declares at least one leaf field (§9.5)" + }, + { + "name": "throws on keyed header without a field list in strict mode", + "input": "m[2:]:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.2", + "note": "A keyed header without a field list declares no cells at all" + }, + { + "name": "throws on keyed marker after the delimiter symbol in strict mode", + "input": "m[2|:]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6", + "note": "The keyed colon must immediately follow the length and precede the delimiter symbol: [2:|]" + }, + { + "name": "throws on keyed marker with leading-zero length in strict mode", + "input": "m[03:]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on whitespace before the keyed marker in strict mode", + "input": "m[2 :]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on explicit comma delimiter after the keyed marker in strict mode", + "input": "m[2:,]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6", + "note": "Comma is never written as a delimiter symbol; [N:] already declares it" + }, + { + "name": "throws on inline content after a keyed header colon in strict mode", + "input": "m[1:]{v}: x\n a: 1", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.2", + "note": "A keyed header carries no inline entries" + }, + { + "name": "throws on a line without an unquoted colon at entry depth in strict mode", + "input": "m[2:]{v}:\n a: 1\n 5", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.2" + }, + { + "name": "throws on duplicate entry keys in strict mode", + "input": "m[2:]{v}:\n a: 1\n a: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.3" + }, + { + "name": "throws on a keyless keyed header as a list item in strict mode", + "input": "items[1]:\n - [2:]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "14.2", + "note": "The keyless keyed header is valid only as the document's root header (§5, §9.5)" + }, + { + "name": "throws on inner array item count not matching its declared length", + "input": "pairs[2]:\n - [3]: 1,2\n - [2]: 3,4", + "expected": null, + "shouldError": true, + "specSection": "9.2" + }, + { + "name": "throws on keyless array header in object field position", + "input": "a:\n [2]: 1,2", + "expected": null, + "shouldError": true, + "specSection": "6" + }, + { + "name": "throws on keyless array header after a depth-0 field", + "input": "a: 1\n[2]: x,y", + "expected": null, + "shouldError": true, + "specSection": "6", + "note": "Keyless headers are valid only as the document's root header or as list items" + }, + { + "name": "throws on keyless fields-bearing header as list item", + "input": "items[1]:\n - [2]{x}:\n 1\n 2", + "expected": null, + "shouldError": true, + "specSection": "6" + }, + { + "name": "throws on inline content after tabular header", + "input": "items[2]{a,b}: 1,2", + "expected": null, + "shouldError": true, + "specSection": "6", + "note": "A fields-bearing header carries no inline content; decoding the values as an inline array would silently drop the field list" + }, + { + "name": "throws on inline content after root tabular header", + "input": "[2]{a,b}: 1,2", + "expected": null, + "shouldError": true, + "specSection": "6" + }, + { + "name": "throws on whitespace between a key and its bracket segment", + "input": "foo [2]: bar,baz", + "expected": null, + "shouldError": true, + "specSection": "6", + "minSpecVersion": "4.1" + }, + { + "name": "throws on duplicate field names in a zero-row tabular header", + "input": "items[0]{a,a}:", + "expected": null, + "shouldError": true, + "specSection": "9.3", + "note": "A duplicate field name is a header defect, diagnosed from the header line alone", + "minSpecVersion": "4.1" + }, + { + "name": "throws on characters after a closing quote in non-strict mode", + "input": "k: \"abc\" def", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "7.4", + "note": "The quoted-token boundary rule applies in strict and non-strict mode alike", + "minSpecVersion": "4.1" + }, + { + "name": "throws on a bare token line inside an array scope in non-strict mode", + "input": "k[1]:\n - x: 1\n bare", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "5.2", + "note": "A scalar line outside root primitive position is an error in strict and non-strict mode alike", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/decode/whitespace.json b/src/test/resources/conformance/decode/whitespace.json index f2c51d24..fc22ed30 100644 --- a/src/test/resources/conformance/decode/whitespace.json +++ b/src/test/resources/conformance/decode/whitespace.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "decode", - "description": "Whitespace tolerance in decoding - surrounding spaces around delimiters and values", + "description": "Whitespace tolerance in decoding – surrounding spaces around delimiters and values", "tests": [ { "name": "tolerates spaces around commas in inline arrays", @@ -10,7 +10,7 @@ "tags": ["a", "b", "c"] }, "specSection": "12", - "note": "Surrounding whitespace SHOULD be tolerated; tokens are trimmed" + "note": "Trimming removes U+0020 only, never other Unicode whitespace" }, { "name": "tolerates spaces around pipes in inline arrays", @@ -30,10 +30,10 @@ }, { "name": "tolerates leading and trailing spaces in tabular row values", - "input": "items[2]{id,name}:\n 1 , Alice \n 2 , Bob ", + "input": "items[2]{id,name}:\n 1 , Ada \n 2 , Bob ", "expected": { "items": [ - { "id": 1, "name": "Alice" }, + { "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" } ] }, @@ -53,8 +53,77 @@ "expected": { "items": ["a", "", "c"] }, + "specSection": "9.1" + }, + { + "name": "preserves NBSP-leading unquoted value", + "input": "k: \u00a0v", + "expected": { + "k": "\u00a0v" + }, + "specSection": "12", + "note": "Token trimming removes U+0020 only; NBSP is part of the token" + }, + { + "name": "preserves NBSP around inline array tokens", + "input": "items[2]: \u00a0a\u00a0,b", + "expected": { + "items": ["\u00a0a\u00a0", "b"] + }, + "specSection": "12", + "note": "A host trim() that strips the full Unicode whitespace set corrupts this round-trip" + }, + { + "name": "decodes CRLF line terminators", + "input": "a: 1\r\nb: 2\r\n", + "expected": { "a": 1, "b": 2 }, + "specSection": "12" + }, + { + "name": "decodes tabular rows with CRLF line terminators", + "input": "items[2]{a,b}:\r\n 1,2\r\n 3,4\r\n", + "expected": { "items": [{ "a": 1, "b": 2 }, { "a": 3, "b": 4 }] }, + "specSection": "12" + }, + { + "name": "keeps an escaped carriage return inside a quoted value", + "input": "a: \"x\\ry\"\r\n", + "expected": { "a": "x\ry" }, + "specSection": "12", + "note": "Only a CR at the end of a line is part of the line terminator" + }, + { + "name": "treats a carriage-return-only line as blank", + "input": "a: 1\n\r\nb: 2", + "expected": { "a": 1, "b": 2 }, + "specSection": "12" + }, + { + "name": "strips a trailing carriage return at end of input", + "input": "a: 1\r", + "expected": { "a": 1 }, + "specSection": "12" + }, + { + "name": "strips a leading byte-order mark", + "input": "\ufeffa: 1", + "expected": { + "a": 1 + }, + "specSection": "12", + "minSpecVersion": "4.1" + }, + { + "name": "treats a hyphen followed by trailing spaces as the bare marker", + "input": "a[1]:\n - ", + "expected": { + "a": [ + {} + ] + }, "specSection": "12", - "note": "Empty token (nothing between delimiters) decodes to empty string" + "note": "Trailing spaces are stripped before line classification", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/encode/arrays-nested.json b/src/test/resources/conformance/encode/arrays-nested.json index f414dc89..69c570ec 100644 --- a/src/test/resources/conformance/encode/arrays-nested.json +++ b/src/test/resources/conformance/encode/arrays-nested.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Nested and mixed array encoding - arrays of arrays, mixed type arrays, root arrays", + "description": "Nested and mixed array encoding – arrays of arrays, mixed type arrays, root arrays", "tests": [ { "name": "encodes nested arrays of primitives", @@ -42,19 +42,19 @@ "specSection": "9.1" }, { - "name": "encodes root-level array of uniform objects in tabular format", + "name": "encodes root-level array of uniform objects in tabular form", "input": [{ "id": 1 }, { "id": 2 }], "expected": "[2]{id}:\n 1\n 2", "specSection": "9.3" }, { - "name": "encodes root-level array of non-uniform objects in list format", + "name": "encodes root-level array of non-uniform objects in list form", "input": [{ "id": 1 }, { "id": 2, "name": "Ada" }], "expected": "[2]:\n - id: 1\n - id: 2\n name: Ada", "specSection": "9.4" }, { - "name": "encodes root-level array mixing primitive, object, and array of objects in list format", + "name": "encodes root-level array mixing primitive, object, and array of objects in list form", "input": ["summary", { "id": 1, "name": "Ada" }, [{ "id": 2 }, { "status": "draft" }]], "expected": "[3]:\n - summary\n - id: 1\n name: Ada\n - [2]:\n - id: 2\n - status: draft", "specSection": "9.4" @@ -86,7 +86,7 @@ "specSection": "8" }, { - "name": "uses list format for arrays mixing primitives and objects", + "name": "uses list form for arrays mixing primitives and objects", "input": { "items": [1, { "a": 1 }, "text"] }, @@ -94,12 +94,39 @@ "specSection": "9.4" }, { - "name": "uses list format for arrays mixing objects and arrays", + "name": "uses list form for arrays mixing objects and arrays", "input": { "items": [{ "a": 1 }, [1, 2]] }, "expected": "items[2]:\n - a: 1\n - [2]: 1,2", "specSection": "9.4" + }, + { + "name": "quotes hash-leading string as list item", + "input": { + "items": ["#x", { "a": 1 }] + }, + "expected": "items[2]:\n - \"#x\"\n - a: 1", + "specSection": "7.2" + }, + { + "name": "uses list form for a tabular-eligible array in list-item position", + "input": { + "a": [ + [ + { + "x": 1 + }, + { + "x": 2 + } + ] + ] + }, + "expected": "a[1]:\n - [2]:\n - x: 1\n - x: 2", + "specSection": "9.3", + "note": "A keyless fields-bearing header is valid only at the document root", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/encode/arrays-objects.json b/src/test/resources/conformance/encode/arrays-objects.json index 2688b85c..360ec23b 100644 --- a/src/test/resources/conformance/encode/arrays-objects.json +++ b/src/test/resources/conformance/encode/arrays-objects.json @@ -1,10 +1,10 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Arrays of objects encoding - list format for non-uniform objects and complex structures", + "description": "Arrays of objects encoding – list form for non-uniform objects and complex structures", "tests": [ { - "name": "uses list format for objects with different fields", + "name": "uses list form for objects with different fields", "input": { "items": [ { "id": 1, "name": "First" }, @@ -15,14 +15,16 @@ "specSection": "9.4" }, { - "name": "uses list format for objects with nested values", + "name": "uses list form for objects with nested values", "input": { "items": [ - { "id": 1, "nested": { "x": 1 } } + { "id": 1, "nested": { "x": 1 } }, + { "id": 2, "nested": { "y": 2 } } ] }, - "expected": "items[1]:\n - id: 1\n nested:\n x: 1", - "specSection": "9.4" + "expected": "items[2]:\n - id: 1\n nested:\n x: 1\n - id: 2\n nested:\n y: 2", + "specSection": "9.4", + "note": "The nested column mixes key sets {x} and {y}, so it is not nested-uniform (§9.3), so the array uses list form" }, { "name": "preserves field order in list items - array first", @@ -41,7 +43,7 @@ "specSection": "10" }, { - "name": "uses list format for objects containing arrays of arrays", + "name": "uses list form for objects containing arrays of arrays", "input": { "items": [ { "matrix": [[1, 2], [3, 4]], "name": "grid" } @@ -51,7 +53,7 @@ "specSection": "10" }, { - "name": "uses tabular format for nested uniform object arrays", + "name": "uses tabular form for nested uniform object arrays", "input": { "items": [ { "users": [{ "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" }], "status": "active" } @@ -62,7 +64,7 @@ "note": "Tabular header on hyphen line with rows at depth +2 and sibling fields at depth +1 (§10)" }, { - "name": "uses list format for nested object arrays with mismatched keys", + "name": "uses list form for nested object arrays with mismatched keys", "input": { "items": [ { "users": [{ "id": 1, "name": "Ada" }, { "id": 2 }], "status": "active" } @@ -72,7 +74,7 @@ "specSection": "10" }, { - "name": "uses list format for objects with multiple array fields", + "name": "uses list form for objects with multiple array fields", "input": { "items": [{ "nums": [1, 2], "tags": ["a", "b"], "name": "test" }] }, @@ -80,7 +82,7 @@ "specSection": "10" }, { - "name": "uses list format for objects with only array fields", + "name": "uses list form for objects with only array fields", "input": { "items": [{ "nums": [1, 2, 3], "tags": ["a", "b"] }] }, @@ -88,7 +90,7 @@ "specSection": "10" }, { - "name": "encodes objects with empty arrays in list format", + "name": "encodes objects with empty arrays in list form", "input": { "items": [ { "name": "Ada", "data": [] } @@ -144,7 +146,7 @@ "specSection": "9.3" }, { - "name": "uses list format when one object has nested field", + "name": "uses list form when one object has nested field", "input": { "items": [ { "id": 1, "data": "string" }, @@ -155,14 +157,35 @@ "specSection": "9.4" }, { - "name": "uses expanded list for arrays containing empty objects", + "name": "uses list form for arrays containing empty objects", "input": { "items": [{}, {}] }, "expected": "items[2]:\n -\n -", "specSection": "9.4", - "minSpecVersion": "3.2", - "note": "Empty objects {} MUST NOT use tabular form per §9.3; encoded via §9.4 expanded list with bare hyphen markers per §10" + "note": "Empty objects are excluded from tabular detection (§9.3), so the array uses list form with bare hyphen markers (§10)" + }, + { + "name": "encodes a keyed-eligible object in a tabular column as a nested field group", + "input": { + "a": [ + { + "t": { + "p": { + "x": 1 + }, + "q": { + "x": 2 + } + }, + "other": 9 + } + ] + }, + "expected": "a[1]{t{p{x},q{x}},other}:\n 1,2,9", + "specSection": "9.5", + "note": "Keyed tabular form is mandated in object-field and root positions, not in a column", + "minSpecVersion": "4.1" } ] } diff --git a/src/test/resources/conformance/encode/arrays-primitive.json b/src/test/resources/conformance/encode/arrays-primitive.json index 1059c605..56440913 100644 --- a/src/test/resources/conformance/encode/arrays-primitive.json +++ b/src/test/resources/conformance/encode/arrays-primitive.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Primitive array encoding - inline arrays of strings, numbers, booleans", + "description": "Primitive array encoding – inline arrays of strings, numbers, booleans", "tests": [ { "name": "encodes string arrays inline", @@ -98,6 +98,14 @@ }, "expected": "items[3]: \"[5]\",\"- item\",\"{key}\"", "specSection": "9.1" + }, + { + "name": "quotes hash-leading string in inline array", + "input": { + "tags": ["#a", "b"] + }, + "expected": "tags[2]: \"#a\",b", + "specSection": "7.2" } ] } diff --git a/src/test/resources/conformance/encode/arrays-tabular.json b/src/test/resources/conformance/encode/arrays-tabular.json index b213e9a8..23650424 100644 --- a/src/test/resources/conformance/encode/arrays-tabular.json +++ b/src/test/resources/conformance/encode/arrays-tabular.json @@ -1,10 +1,10 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Tabular array encoding - arrays of uniform objects with primitive values", + "description": "Tabular array encoding – arrays of uniform objects with primitive values", "tests": [ { - "name": "encodes arrays of uniform objects in tabular format", + "name": "encodes arrays of uniform objects in tabular form", "input": { "items": [ { "sku": "A1", "qty": 2, "price": 9.99 }, @@ -15,7 +15,7 @@ "specSection": "9.3" }, { - "name": "encodes null values in tabular format", + "name": "encodes null values in tabular form", "input": { "items": [ { "id": 1, "value": null }, @@ -68,6 +68,115 @@ }, "expected": "\"\"[2]{id,name}:\n 1,Ada\n 2,Bob", "specSection": "9.3" + }, + { + "name": "quotes hash-leading string in tabular cell", + "input": { + "items": [{ "tag": "#a" }, { "tag": "b" }] + }, + "expected": "items[2]{tag}:\n \"#a\"\n b", + "specSection": "7.2", + "note": "Unquoted, the first row would read as a comment line on decode" + }, + { + "name": "collapses a uniform nested object column into a nested field group", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, + { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + ] + }, + "expected": "orders[2]{id,customer{name,country},total}:\n 1,Ada,DK,99\n 2,Bob,UK,149", + "specSection": "9.3" + }, + { + "name": "collapses sibling nested field groups with depth-first row layout", + "input": { + "shipments": [ + { "id": "s1", "sender": { "name": "ACME", "city": "Berlin" }, "receiver": { "name": "Globex", "city": "Oslo" } } + ] + }, + "expected": "shipments[1]{id,sender{name,city},receiver{name,city}}:\n s1,ACME,Berlin,Globex,Oslo", + "specSection": "9.3" + }, + { + "name": "collapses nested field groups recursively without a depth cap", + "input": { + "items": [ + { "id": 1, "geo": { "point": { "lat": 1.5, "lon": 2.5 } } }, + { "id": 2, "geo": { "point": { "lat": 3, "lon": 4 } } } + ] + }, + "expected": "items[2]{id,geo{point{lat,lon}}}:\n 1,1.5,2.5\n 2,3,4", + "specSection": "9.3" + }, + { + "name": "uses the active delimiter inside nested field groups", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, + { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + ] + }, + "options": { "delimiter": "|" }, + "expected": "orders[2|]{id|customer{name|country}|total}:\n 1|Ada|DK|99\n 2|Bob|UK|149", + "specSection": "9.3" + }, + { + "name": "quotes subfield names inside nested field groups per key encoding", + "input": { + "items": [ + { "id": 1, "customer": { "full name": "Ada", "country": "UK" } } + ] + }, + "expected": "items[1]{id,customer{\"full name\",country}}:\n 1,Ada,UK", + "specSection": "9.3" + }, + { + "name": "falls back to list form when nested object keys differ per row", + "input": { + "entries": [ + { "id": 1, "meta": { "a": 1 } }, + { "id": 2, "meta": { "b": 2 } } + ] + }, + "expected": "entries[2]:\n - id: 1\n meta:\n a: 1\n - id: 2\n meta:\n b: 2", + "specSection": "9.3", + "note": "The meta column is not nested-uniform (differing key sets), so the whole array uses §9.4" + }, + { + "name": "falls back to list form when a column mixes null and objects", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada" } }, + { "id": 2, "customer": null } + ] + }, + "expected": "orders[2]:\n - id: 1\n customer:\n name: Ada\n - id: 2\n customer: null", + "specSection": "9.3", + "note": "A null cell is a primitive, so the customer column is neither uniform-primitive nor nested-uniform" + }, + { + "name": "falls back to list form when a nested object contains an array", + "input": { + "orders": [ + { "id": 1, "customer": { "name": "Ada", "tags": ["x"] } } + ] + }, + "expected": "orders[1]:\n - id: 1\n customer:\n name: Ada\n tags[1]: x", + "specSection": "9.3", + "note": "Array values disqualify a column; rows must contain only primitive cells" + }, + { + "name": "falls back to list form when a nested column contains an empty object", + "input": { + "items": [ + { "id": 1, "meta": {} } + ] + }, + "expected": "items[1]:\n - id: 1\n meta:", + "specSection": "9.3", + "note": "Nested-uniform requires non-empty objects, mirroring the empty-object exclusion for elements" } ] } diff --git a/src/test/resources/conformance/encode/delimiters.json b/src/test/resources/conformance/encode/delimiters.json index 5079916b..03c9271d 100644 --- a/src/test/resources/conformance/encode/delimiters.json +++ b/src/test/resources/conformance/encode/delimiters.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Delimiter options - tab and pipe delimiters, delimiter-aware quoting", + "description": "Delimiter options – tab and pipe delimiters, delimiter-aware quoting", "tests": [ { "name": "encodes primitive arrays with tab delimiter", diff --git a/src/test/resources/conformance/encode/key-folding.json b/src/test/resources/conformance/encode/key-folding.json deleted file mode 100644 index a96b34f7..00000000 --- a/src/test/resources/conformance/encode/key-folding.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "version": "1.5", - "category": "encode", - "description": "Key folding with safe mode, depth control, collision avoidance", - "tests": [ - { - "name": "encodes folded chain to primitive (safe mode)", - "input": { - "a": { - "b": { - "c": 1 - } - } - }, - "expected": "a.b.c: 1", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "encodes folded chain with inline array", - "input": { - "data": { - "meta": { - "items": ["x", "y"] - } - } - }, - "expected": "data.meta.items[2]: x,y", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "encodes folded chain with tabular array", - "input": { - "a": { - "b": { - "items": [ - { "id": 1, "name": "A" }, - { "id": 2, "name": "B" } - ] - } - } - }, - "expected": "a.b.items[2]{id,name}:\n 1,A\n 2,B", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "skips folding when segment requires quotes (safe mode)", - "input": { - "data": { - "full-name": { - "x": 1 - } - } - }, - "expected": "data:\n \"full-name\":\n x: 1", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "skips folding on sibling literal-key collision (safe mode)", - "input": { - "data": { - "meta": { - "items": [1, 2] - } - }, - "data.meta.items": "literal" - }, - "expected": "data:\n meta:\n items[2]: 1,2\ndata.meta.items: literal", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4", - "note": "Collision avoidance: folding would create duplicate key" - }, - { - "name": "encodes partial folding with flattenDepth=2", - "input": { - "a": { - "b": { - "c": { - "d": 1 - } - } - } - }, - "expected": "a.b:\n c:\n d: 1", - "options": { - "keyFolding": "safe", - "flattenDepth": 2 - }, - "specSection": "13.4" - }, - { - "name": "encodes full chain with flattenDepth=Infinity (default)", - "input": { - "a": { - "b": { - "c": { - "d": 1 - } - } - } - }, - "expected": "a.b.c.d: 1", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "encodes standard nesting with flattenDepth=0 (no folding)", - "input": { - "a": { - "b": { - "c": 1 - } - } - }, - "expected": "a:\n b:\n c: 1", - "options": { - "keyFolding": "safe", - "flattenDepth": 0 - }, - "specSection": "13.4", - "note": "flattenDepth below 2 has no practical effect, leaving standard nesting (§13.4)" - }, - { - "name": "encodes standard nesting with keyFolding=off (baseline)", - "input": { - "a": { - "b": { - "c": 1 - } - } - }, - "expected": "a:\n b:\n c: 1", - "options": { - "keyFolding": "off" - }, - "specSection": "13.4" - }, - { - "name": "encodes folded chain ending with empty object", - "input": { - "a": { - "b": { - "c": {} - } - } - }, - "expected": "a.b.c:", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "stops folding at array boundary (not single-key object)", - "input": { - "a": { - "b": [1, 2] - } - }, - "expected": "a.b[2]: 1,2", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - }, - { - "name": "encodes folded chains preserving sibling field order", - "input": { - "first": { - "second": { - "third": 1 - } - }, - "simple": 2, - "short": { - "path": 3 - } - }, - "expected": "first.second.third: 1\nsimple: 2\nshort.path: 3", - "options": { - "keyFolding": "safe" - }, - "specSection": "13.4" - } - ] -} diff --git a/src/test/resources/conformance/encode/objects-keyed.json b/src/test/resources/conformance/encode/objects-keyed.json new file mode 100644 index 00000000..7fd4e879 --- /dev/null +++ b/src/test/resources/conformance/encode/objects-keyed.json @@ -0,0 +1,155 @@ +{ + "version": "4.0", + "category": "encode", + "description": "Keyed tabular encoding – objects whose values are uniform objects collapse into keyed headers with entry rows", + "tests": [ + { + "name": "encodes objects of uniform objects in keyed tabular form", + "input": { + "servers": { + "alpha": { "host": "a.example.com", "port": 8080 }, + "beta": { "host": "b.example.com", "port": 9090 } + } + }, + "expected": "servers[2:]{host,port}:\n alpha: a.example.com,8080\n beta: b.example.com,9090", + "specSection": "9.5" + }, + { + "name": "encodes an eligible root object with a keyless keyed header", + "input": { + "alice": { "age": 30, "city": "Berlin" }, + "bob": { "age": 25, "city": "Oslo" } + }, + "expected": "[2:]{age,city}:\n alice: 30,Berlin\n bob: 25,Oslo", + "specSection": "9.5", + "note": "The keyless keyed header is valid only at the root (§5)" + }, + { + "name": "collapses uniform nested object columns inside keyed headers", + "input": { + "regions": { + "eu": { "name": "Europe", "geo": { "lat": 50, "lon": 10 } }, + "us": { "name": "America", "geo": { "lat": 40, "lon": -100 } } + } + }, + "expected": "regions[2:]{name,geo{lat,lon}}:\n eu: Europe,50,10\n us: America,40,-100", + "specSection": "9.5" + }, + { + "name": "orders fields by the first entry value's encounter order", + "input": { + "m": { + "a": { "x": 1, "y": 2 }, + "b": { "y": 4, "x": 3 } + } + }, + "expected": "m[2:]{x,y}:\n a: 1,2\n b: 3,4", + "specSection": "9.5" + }, + { + "name": "uses the active delimiter in keyed headers and entry-row cells", + "input": { + "servers": { + "alpha": { "host": "a.example.com", "port": 8080 }, + "beta": { "host": "b.example.com", "port": 9090 } + } + }, + "options": { "delimiter": "|" }, + "expected": "servers[2:|]{host|port}:\n alpha: a.example.com|8080\n beta: b.example.com|9090", + "specSection": "9.5", + "note": "The keyed marker precedes the delimiter symbol: [2:|]" + }, + { + "name": "quotes entry keys per key encoding", + "input": { + "ids": { + "42": { "v": 1 }, + "my key": { "v": 2 } + } + }, + "expected": "ids[2:]{v}:\n \"42\": 1\n \"my key\": 2", + "specSection": "9.5", + "note": "Entry keys follow §7.3, the same rule as object keys" + }, + { + "name": "quotes entry-row cells containing the active delimiter", + "input": { + "notes": { + "n1": { "text": "a,b" }, + "n2": { "text": "c" } + } + }, + "expected": "notes[2:]{text}:\n n1: \"a,b\"\n n2: c", + "specSection": "9.5" + }, + { + "name": "keeps single-entry objects in nested form", + "input": { + "config": { + "only": { "a": 1, "b": 2 } + } + }, + "expected": "config:\n only:\n a: 1\n b: 2", + "specSection": "9.5", + "note": "Keyed tabular detection requires at least two entries" + }, + { + "name": "keeps objects in nested form when entry values have differing key sets", + "input": { + "envs": { + "dev": { "host": "x" }, + "prod": { "host": "y", "port": 1 } + } + }, + "expected": "envs:\n dev:\n host: x\n prod:\n host: y\n port: 1", + "specSection": "9.5" + }, + { + "name": "keeps objects in nested form when a value is primitive", + "input": { + "m": { + "a": { "x": 1 }, + "b": 2 + } + }, + "expected": "m:\n a:\n x: 1\n b: 2", + "specSection": "9.5" + }, + { + "name": "keeps objects in nested form when an entry value contains an array", + "input": { + "m": { + "a": { "tags": ["x"] }, + "b": { "tags": ["y"] } + } + }, + "expected": "m:\n a:\n tags[1]: x\n b:\n tags[1]: y", + "specSection": "9.5", + "note": "Entry-row cells must be primitive leaves; an array value disqualifies the column" + }, + { + "name": "emits a keyed header on the hyphen line when it is the first field of a list item", + "input": { + "items": [ + { "config": { "a": { "x": 1 }, "b": { "x": 2 } }, "status": "ok" }, + { "status": "down" } + ] + }, + "expected": "items[2]:\n - config[2:]{x}:\n a: 1\n b: 2\n status: ok\n - status: down", + "specSection": "10", + "note": "Entry rows at depth +2, sibling fields at depth +1, mirroring tabular arrays on hyphen lines" + }, + { + "name": "never encodes an anonymous array element in keyed tabular form", + "input": { + "items": [ + { "a": { "x": 1 }, "b": { "x": 2 } }, + 5 + ] + }, + "expected": "items[2]:\n - a:\n x: 1\n b:\n x: 2\n - 5", + "specSection": "9.5", + "note": "The element object is keyed-eligible but has no key; the keyless form is root-only, so it encodes per §10" + } + ] +} diff --git a/src/test/resources/conformance/encode/objects.json b/src/test/resources/conformance/encode/objects.json index 5a262041..a63f92c6 100644 --- a/src/test/resources/conformance/encode/objects.json +++ b/src/test/resources/conformance/encode/objects.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Object encoding - simple objects, nested objects, key encoding", + "description": "Object encoding – simple objects, nested objects, key encoding", "tests": [ { "name": "preserves key order in objects", @@ -172,6 +172,23 @@ "expected": "\"\": 1", "specSection": "7.3" }, + { + "name": "quotes non-ASCII key", + "input": { + "café": 1 + }, + "expected": "\"café\": 1", + "specSection": "7.3", + "note": "The unquoted-key pattern is ASCII-only, so every non-ASCII key is quoted" + }, + { + "name": "quotes CJK key", + "input": { + "名前": "x" + }, + "expected": "\"名前\": x", + "specSection": "7.3" + }, { "name": "escapes newline in key", "input": { @@ -202,8 +219,7 @@ "a\u0004b": 1 }, "expected": "\"a\\u0004b\": 1", - "specSection": "7.1", - "minSpecVersion": "3.1" + "specSection": "7.1" }, { "name": "escapes U+001F control character in key via \\uXXXX", @@ -211,8 +227,7 @@ "x\u001fy": 2 }, "expected": "\"x\\u001fy\": 2", - "specSection": "7.1", - "minSpecVersion": "3.1" + "specSection": "7.1" }, { "name": "encodes deeply nested objects", @@ -233,6 +248,50 @@ }, "expected": "user:", "specSection": "8" + }, + { + "name": "encodes __proto__ own property as an ordinary key", + "input": { + "__proto__": "polluted", + "safe": true + }, + "expected": "__proto__: polluted\nsafe: true", + "specSection": "15", + "note": "No key has special meaning, so the encoder emits prototype-named own entries like any other (§8)" + }, + { + "name": "encodes constructor and prototype own properties as ordinary keys", + "input": { + "constructor": 1, + "prototype": 2 + }, + "expected": "constructor: 1\nprototype: 2", + "specSection": "15" + }, + { + "name": "encodes __proto__ as a tabular field name", + "input": { + "rows": [ + { + "__proto__": "a", + "x": 1 + }, + { + "__proto__": "b", + "x": 2 + } + ] + }, + "expected": "rows[2]{__proto__,x}:\n a,1\n b,2", + "specSection": "15" + }, + { + "name": "quotes hash-leading string in object field value", + "input": { + "note": "#x" + }, + "expected": "note: \"#x\"", + "specSection": "7.2" } ] } diff --git a/src/test/resources/conformance/encode/primitives.json b/src/test/resources/conformance/encode/primitives.json index 373f536b..83cc7823 100644 --- a/src/test/resources/conformance/encode/primitives.json +++ b/src/test/resources/conformance/encode/primitives.json @@ -1,7 +1,7 @@ { - "version": "3.1", + "version": "4.0", "category": "encode", - "description": "Primitive value encoding - strings, numbers, booleans, null", + "description": "Primitive value encoding – strings, numbers, booleans, null", "tests": [ { "name": "encodes safe strings without quotes", @@ -25,8 +25,7 @@ "name": "quotes string that looks like true", "input": "true", "expected": "\"true\"", - "specSection": "7.2", - "note": "String representation of boolean must be quoted" + "specSection": "7.2" }, { "name": "quotes string that looks like false", @@ -99,15 +98,13 @@ "name": "quotes string with array-like syntax", "input": "[3]: x,y", "expected": "\"[3]: x,y\"", - "specSection": "7.2", - "note": "Looks like array header" + "specSection": "7.2" }, { "name": "quotes string starting with hyphen-space", "input": "- item", "expected": "\"- item\"", - "specSection": "7.2", - "note": "Looks like list item marker" + "specSection": "7.2" }, { "name": "quotes single hyphen as object value", @@ -170,6 +167,13 @@ "expected": "hello 👋 world", "specSection": "7.2" }, + { + "name": "encodes emoji inside a quoted string", + "input": "a,🚀", + "expected": "\"a,🚀\"", + "specSection": "7.1", + "note": "Quoting is triggered by the delimiter; the supplementary scalar stays literal UTF-8, never a surrogate escape" + }, { "name": "encodes positive integer", "input": 42, @@ -201,25 +205,24 @@ "specSection": "2" }, { - "name": "encodes scientific notation as decimal", + "name": "encodes large integer without exponent notation", "input": 1000000, "expected": "1000000", "specSection": "2", - "note": "1e6 input, but represented as decimal" + "note": "Within the canonical decimal range" }, { - "name": "encodes small decimal from scientific notation", + "name": "encodes small decimal without exponent notation", "input": 0.000001, "expected": "0.000001", "specSection": "2", - "note": "1e-6 input" + "note": "Lower bound of the canonical decimal range" }, { "name": "encodes large number", "input": 100000000000000000000, "expected": "100000000000000000000", - "specSection": "2", - "note": "1e20" + "specSection": "2" }, { "name": "encodes MAX_SAFE_INTEGER", @@ -231,8 +234,7 @@ "name": "encodes repeating decimal with full precision", "input": 0.3333333333333333, "expected": "0.3333333333333333", - "specSection": "2", - "note": "Result of 1/3 in JavaScript" + "specSection": "2" }, { "name": "encodes true", @@ -251,6 +253,26 @@ "input": null, "expected": "null", "specSection": "2" + }, + { + "name": "quotes leading-plus numeric-like string", + "input": "+1", + "expected": "\"+1\"", + "specSection": "7.2", + "note": "Unquoted, a v3 host-parser decoder would read the number 1" + }, + { + "name": "quotes string equal to hash", + "input": "#", + "expected": "\"#\"", + "specSection": "7.2", + "note": "Comment marker at position 0 must be quoted" + }, + { + "name": "quotes string starting with hash", + "input": "#hello", + "expected": "\"#hello\"", + "specSection": "7.2" } ] } diff --git a/src/test/resources/conformance/encode/whitespace.json b/src/test/resources/conformance/encode/whitespace.json index 27bb3a03..1c6eae33 100644 --- a/src/test/resources/conformance/encode/whitespace.json +++ b/src/test/resources/conformance/encode/whitespace.json @@ -1,7 +1,7 @@ { - "version": "1.4", + "version": "4.0", "category": "encode", - "description": "Whitespace and formatting invariants - no trailing spaces, no trailing newlines", + "description": "Whitespace and formatting invariants – no trailing spaces, no trailing newlines", "tests": [ { "name": "produces no trailing newline at end of output", @@ -21,11 +21,10 @@ "items": ["a", "b"] }, "expected": "user:\n id: 123\n name: Ada\nitems[2]: a,b", - "specSection": "12", - "note": "2-space indentation, no trailing spaces on any line" + "specSection": "12" }, { - "name": "respects custom indent size option", + "name": "respects custom indentSize option", "input": { "user": { "name": "Ada", @@ -35,8 +34,18 @@ "expected": "user:\n name: Ada\n role: admin", "specSection": "12", "options": { - "indent": 4 + "indentSize": 4 } + }, + { + "name": "leaves non-ASCII whitespace unquoted", + "input": { + "k": "\u00a0x\u00a0" + }, + "expected": "k: \u00a0x\u00a0", + "specSection": "7.2", + "note": "The whitespace trigger is U+0020 and U+0009 only", + "minSpecVersion": "4.1" } ] }