From 939211a05e51ec0a1dbecd545492f32e91a455c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 3 Aug 2026 06:27:50 +0200 Subject: [PATCH] fix: read local files without Hadoop in COPY INTO (JDK 23+) Every local file read in COPY INTO went through Hadoop, whose UserGroupInformation.getCurrentUser() calls Subject.getSubject. JDK 23 re-specified that method to throw by default, and JEP 486 (JDK 24) made it throw unconditionally with no escape hatch, so COPY INTO from a local file failed on any modern JVM (reported from DBeaver's bundled Temurin 25). Add LocalPath, a pure classifier with zero Hadoop imports, plus an openStream dispatcher and a validateFile split. Local paths (schemeless, or file: with an empty/localhost authority) are now opened through java.nio.file and never touch Hadoop; every other scheme is unchanged. Seven Hadoop entry points were rewired, not the five the issue listed - the two missing ones were FileSource.validateFile (which runs before every read, so a five-site fix would still have thrown) and FileFormatDetector.isDeltaTable. Parquet, Delta Lake and all remote schemes stay Hadoop-bound on purpose and remain unsupported on JDK 23+; that boundary is now documented per source with a three-column JDK matrix rather than the misleading "JDK 24+". Also unifies scheme detection: HadoopConfigurationFactory.forPath now uses LocalPath.scheme instead of its own Try(new URI(path).getScheme), fixing two silent credential losses - S3A://bucket/x (uppercase matched no lowercase literal) and s3a://bucket/my file.jsonl (new URI throws on the space) both fell through to localConf() with no S3 credentials. Regression guards are JDK-independent and armed: a Configuration whose fs.file.impl is unresolvable (with fs.file.impl.disable.cache, without which Hadoop's static cache makes the guard vacuous) proves Hadoop was not consulted, and each suite carries a self-check asserting the poisoning really throws. Verified: 67/67 on zulu-11/17/21; on zulu-25 LocalPathSpec 41/41 and FileSourceSpec 19 pass with only the 7 documented Parquet/Delta tests failing with "getSubject is not supported" - which doubles as an in-suite control. Probe183 on JDK 25 confirms the same JVM reproduces the bug before the fixed path reads the file. ES8 integration 56/56 against live ES 8.18.3. Closed Issue #183 --- core/build.sbt | 13 + .../elastic/client/file/LocalPath.scala | 151 +++++++ .../elastic/client/file/package.scala | 125 ++++-- .../elastic/client/file/FileSourceSpec.scala | 130 ++++++ .../elastic/client/file/LocalPathSpec.scala | 412 ++++++++++++++++++ documentation/sql/dml_statements.md | 53 +++ documentation/sql/known_limitations.md | 10 + .../repl/ReplGatewayIntegrationSpec.scala | 52 +++ 8 files changed, 920 insertions(+), 26 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/file/LocalPath.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/file/LocalPathSpec.scala diff --git a/core/build.sbt b/core/build.sbt index 2c09af23..058a2a5d 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -66,3 +66,16 @@ json4s ++ mockito ++ avro ++ cloudConnectors ++ repl :+ "com.google.code.gson" % "com.typesafe.scala-logging" %% "scala-logging" % Versions.scalaLogging :+ "io.delta" %% "delta-standalone" % Versions.delta :+ "org.scalatest" %% "scalatest" % Versions.scalatest % Test + +// Issue #183: run the very same test suite on an arbitrary JDK without changing the compile JDK. +// sbt -Dtest.jdk.home=/Library/Java/JavaVirtualMachines/zulu-25.jdk/Contents/Home \ +// "core/testOnly *LocalPathSpec *FileSourceSpec" +// With the property unset this is a no-op: tests run in-process on the default JDK as before. +Test / fork := sys.props.get("test.jdk.home").isDefined + +Test / javaHome := sys.props.get("test.jdk.home").map(file) + +// `Test / parallelExecution := false` at build.sbt:101 is a BARE top-level statement, i.e. scoped +// to the ROOT project only — it is NOT inherited by `core`. Without this line `core`'s suites run +// concurrently in one JVM and the issue-#183 guards become order-dependent (see LocalPathSpec). +Test / parallelExecution := false diff --git a/core/src/main/scala/app/softnetwork/elastic/client/file/LocalPath.scala b/core/src/main/scala/app/softnetwork/elastic/client/file/LocalPath.scala new file mode 100644 index 00000000..8f706517 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/file/LocalPath.scala @@ -0,0 +1,151 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.file + +import java.net.URI +import java.nio.file.{Path, Paths} + +import scala.util.{Success, Try} +import scala.util.matching.Regex + +/** Classifies a `COPY INTO … FROM ''` source string as "on this machine's filesystem" or + * "somewhere Hadoop has to reach". + * + * Why this exists (issue #183): Hadoop's `FileSystem.get` reaches + * `UserGroupInformation.getCurrentUser()`, which calls + * `javax.security.auth.Subject.getSubject(AccessControlContext)`. JDK 23 re-specified that method + * to throw `UnsupportedOperationException` whenever a Security Manager is not allowed (the + * default), and JEP 486 (JDK 24) made it throw `("getSubject is not supported")` unconditionally + * while removing the `-Djava.security.manager=allow` escape hatch (the VM refuses to start with + * it). Verified against hadoop-common 3.4.2 bytecode — upgrading Hadoop does not help. + * + * Anything that only needs bytes off a local file therefore bypasses Hadoop entirely. Remote + * schemes (`s3a`, `s3`, `gs`, `abfs`, `abfss`, `wasb`, `wasbs`, `hdfs`, …) keep going through + * Hadoop unchanged, and so do Parquet and Delta reads, which genuinely need it. + * + * This object intentionally has NO Hadoop dependency so it can be unit-tested in isolation. + */ +object LocalPath { + + /** A URI scheme is at least TWO characters before the colon. + * + * The lower bound is not cosmetic: `new URI("C:/data/x.jsonl")` parses `C` as a scheme, so a + * one-character prefix must never be treated as one — otherwise every Windows drive-letter path + * would be misrouted to Hadoop. + */ + private val SchemePrefix: Regex = """^([A-Za-z][A-Za-z0-9+.-]+):""".r + + private val FileScheme = "file" + + /** Only these authorities denote "this machine". */ + private val LocalAuthorities = Set("", "localhost") + + /** Extracted so `LocalPathSpec` can exercise BOTH platforms deterministically on either OS. */ + private[file] val onWindows: Boolean = java.io.File.separatorChar == '\\' + + /** The URI scheme of `filePath`, lowercased, or `None` when it has none. + * + * This is the ONLY scheme parser in the file package. `HadoopConfigurationFactory.forPath` used + * to run its own (`Try(new URI(path).getScheme)`), which disagreed with this one in two ways + * that silently cost a user their credentials — see AD-10. + * + * A Windows drive letter is not a scheme: [[SchemePrefix]] requires at least two characters + * before the colon, so `C:/data/x.jsonl` yields `None`. + */ + def scheme(filePath: String): Option[String] = + Option(filePath) + .filter(_.trim.nonEmpty) + .flatMap(raw => SchemePrefix.findFirstMatchIn(raw).map(_.group(1).toLowerCase)) + + /** Returns the local [[java.nio.file.Path]] denoted by `filePath`, or `None` when the path is not + * on this machine's filesystem and must be handled by Hadoop. + * + * Local means: no scheme at all (absolute, relative, or a Windows drive path), or the `file:` + * scheme with an empty or `localhost` authority. + * + * `~` is NOT expanded, and surrounding whitespace is NOT trimmed — both match Hadoop's + * `Path(String)` exactly. Pass an absolute path. + * + * Throws [[java.nio.file.InvalidPathException]] (a subclass of `IllegalArgumentException`) for a + * string this platform cannot represent as a path at all — see AD-8. + */ + def resolve(filePath: String): Option[Path] = + // `filter(_.trim.nonEmpty)` rejects blank input WITHOUT rewriting the string: Hadoop preserves + // leading/trailing whitespace in a file name and so must we, or `COPY INTO … FROM '/tmp/x '` + // silently reads `/tmp/x`. + Option(filePath).filter(_.trim.nonEmpty).flatMap { raw => + scheme(raw) match { + case None => Some(Paths.get(raw)) + case Some(FileScheme) => fromFileUri(raw) // `scheme` already lowercased it + case Some(_) => None // remote scheme → Hadoop + } + } + + private def fromFileUri(raw: String): Option[Path] = { + // `new URI` percent-decodes correctly (UTF-8, and unlike URLDecoder it does not turn '+' into a + // space) but rejects unencoded characters such as a literal space. When it rejects the input, + // the user did not percent-encode, so the remainder is already the literal path. + // + // A query or fragment disqualifies the URI reading. Hadoop's `Path(String)` takes "the rest of + // the string" as the path — "query & fragment not supported" — so `?` and `#` are ordinary file + // name characters to it (verified: `new Path("file:///a/report#1.jsonl").toUri.getPath` is + // `/a/report#1.jsonl`). Using `u.getPath` there would truncate to `/a/report` and silently read + // a DIFFERENT file. Fall through to the literal split, which keeps them. + val (authority, path) = Try(new URI(raw)) match { + case Success(u) + if u.getPath != null && u.getPath.nonEmpty && + u.getQuery == null && u.getFragment == null => + (Option(u.getAuthority).getOrElse(""), u.getPath) + case _ => + // Also the branch for an OPAQUE `file:` URI (`file:relative/x.jsonl`), where `getPath` is + // null. Opaque URIs are taken literally — they are not percent-decoded. + splitLiteral(raw.substring(FileScheme.length + 1)) + } + + // Only an empty authority or `localhost` denotes this machine. Any other host keeps the + // pre-existing Hadoop behaviour rather than silently reinterpreting it as a local path. + if (!LocalAuthorities.contains(authority.toLowerCase)) None + else Option(path).filter(_.nonEmpty).map(p => Paths.get(stripDriveSlash(p, onWindows))) + } + + /** Splits `//authority/path`, `///path` or `/path` (the part after `file:`) without URI parsing. + */ + private def splitLiteral(rest: String): (String, String) = + if (rest.startsWith("//")) { + val afterSlashes = rest.substring(2) + afterSlashes.indexOf('/') match { + case -1 => (afterSlashes, "") + case idx => (afterSlashes.substring(0, idx), afterSlashes.substring(idx)) + } + } else ("", rest) + + /** `file:///C:/data/x.jsonl` yields the URI path `/C:/data/x.jsonl`; Windows needs the leading + * slash removed before `Paths.get` will accept it. + * + * Gated on the platform on purpose: on POSIX `/C:` is a perfectly legal directory name, and + * stripping the slash there would turn an absolute path into a CWD-relative one. + * + * `windows` is a parameter rather than a direct read of [[onWindows]] so both branches are + * unit-testable on either OS. + */ + private[file] def stripDriveSlash(p: String, windows: Boolean): String = + if ( + windows && p.length >= 3 && p.charAt(0) == '/' && p.charAt(2) == ':' && + Character.isLetter(p.charAt(1)) + ) p.substring(1) + else p +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala index 3da11794..ba846598 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala @@ -32,10 +32,10 @@ import org.apache.parquet.hadoop.util.HadoopInputFile import io.delta.standalone.DeltaLog import io.delta.standalone.data.{CloseableIterator, RowRecord} import io.delta.standalone.types._ -import org.apache.parquet.io.SeekableInputStream import org.slf4j.{Logger, LoggerFactory} -import java.io.{BufferedReader, InputStream, InputStreamReader} +import java.io.{BufferedInputStream, BufferedReader, InputStream, InputStreamReader} +import java.nio.file.{Files, Path => NioPath} import scala.concurrent.{blocking, ExecutionContext, Future} import scala.io.{Source => IoSource} import scala.util.{Failure, Success, Try} @@ -56,6 +56,33 @@ package object file { conf } + /** Default read-ahead buffer, mirroring the `io.file.buffer.size` that [[hadoopConfiguration]] + * and `HadoopConfigurationFactory.base()` already set (64 KB). + */ + private val DefaultBufferSize = 65536 + + /** Opens `filePath` for reading. + * + * Local paths (schemeless, or `file:` with an empty/`localhost` authority) are opened straight + * through `java.nio.file`, never touching Hadoop — see [[LocalPath]] and issue #183 + * (`Subject.getSubject` throws on JDK 23+ by default and on JDK 24+ unconditionally, so Hadoop's + * `UserGroupInformation.getCurrentUser()` can no longer be called). + * + * Every other scheme keeps the pre-existing Hadoop path unchanged. + */ + def openStream(filePath: String)(implicit conf: Configuration): InputStream = + LocalPath.resolve(filePath) match { + case Some(local) => + // Hadoop's LocalFileSystem wraps its stream in a BufferedFSInputStream sized by + // `io.file.buffer.size`; `Files.newInputStream` is unbuffered, so without this every + // 8 KB BufferedReader/Jackson refill becomes a syscall. Preserve the tuned buffer. + new BufferedInputStream( + Files.newInputStream(local), + conf.getInt("io.file.buffer.size", DefaultBufferSize) + ) + case None => HadoopInputFile.fromPath(new Path(filePath), conf).newStream() + } + /** Base trait for file sources */ sealed trait FileSource { @@ -146,6 +173,46 @@ package object file { protected def validateFile( filePath: String, checkIsFile: Boolean = true + )(implicit conf: Configuration): Unit = + LocalPath.resolve(filePath) match { + case Some(local) => validateLocalPath(filePath, local, checkIsFile) + case None => validateHadoopPath(filePath, checkIsFile) + } + + /** Local fast path — never touches Hadoop (issue #183). Error messages and log lines are + * byte-for-byte identical to the Hadoop branch: `FileSourceSpec` asserts on them. + */ + private def validateLocalPath( + filePath: String, + local: NioPath, + checkIsFile: Boolean + ): Unit = { + if (!Files.exists(local)) { + throw new IllegalArgumentException(s"File does not exist: $filePath") + } + + if (checkIsFile && !Files.isRegularFile(local)) { + throw new IllegalArgumentException(s"Path is not a file: $filePath") + } + + if (!checkIsFile && !Files.isDirectory(local)) { + throw new IllegalArgumentException(s"Path is not a directory: $filePath") + } + + val length = if (checkIsFile) Files.size(local) else 0L + + if (checkIsFile && length == 0) { + logger.warn(s"⚠️ File is empty: $filePath") + } + + val pathType = if (checkIsFile) "file" else "directory" + val sizeInfo = if (checkIsFile) s"($length bytes)" else "" + logger.info(s"📁 Loading $pathType: $filePath $sizeInfo") + } + + private def validateHadoopPath( + filePath: String, + checkIsFile: Boolean )(implicit conf: Configuration): Unit = { val path = new Path(filePath) val fs = FileSystem.get(path.toUri, conf) @@ -335,7 +402,7 @@ package object file { create = () => { logger.info(s"📂 Opening JSON file: $filePath") Try { - val is: InputStream = HadoopInputFile.fromPath(new Path(filePath), conf).newStream() + val is: InputStream = openStream(filePath) new BufferedReader(new InputStreamReader(is, "UTF-8")) } match { case Success(reader) => reader @@ -430,14 +497,13 @@ package object file { Source .unfoldResource[String, (InputStream, JsonParser)]( - // Create: Open file via Hadoop and create JSON parser + // Create: Open file and create JSON parser create = () => { - logger.info(s"📂 Opening JSON Array file via Hadoop: $filePath") + logger.info(s"📂 Opening JSON Array file: $filePath") Try { - val is: SeekableInputStream = - HadoopInputFile.fromPath(new Path(filePath), conf).newStream() + val is: InputStream = openStream(filePath) - // Create Jackson parser on top of Hadoop SeekableInputStream + // Create Jackson parser on top of the input stream val parser = jsonFactory.createParser(is) // Expect array start @@ -449,7 +515,7 @@ package object file { ) } - logger.info(s"📊 Started parsing JSON Array via Hadoop FS") + logger.info(s"📊 Started parsing JSON Array") (is, parser) } match { case Success(result) => result @@ -499,7 +565,7 @@ package object file { } }, - // Close: Close parser and Hadoop input stream + // Close: Close parser and input stream close = { case (inputStream, parser) => Try { parser.close() // This also closes the underlying stream @@ -510,12 +576,12 @@ package object file { logger.warn(s"⚠️ Failed to close JSON Array parser: ${ex.getMessage}") } - // Ensure Hadoop stream is closed + // Ensure the stream is closed Try(inputStream.close()) match { case Success(_) => - logger.debug(s"🔒 Closed Hadoop input stream for: $filePath") + logger.debug(s"🔒 Closed input stream for: $filePath") case Failure(ex) => - logger.warn(s"⚠️ Failed to close Hadoop input stream: ${ex.getMessage}") + logger.warn(s"⚠️ Failed to close input stream: ${ex.getMessage}") } } ) @@ -544,7 +610,7 @@ package object file { Source .future(Future { blocking { - val is: InputStream = HadoopInputFile.fromPath(new Path(filePath), conf).newStream() + val is: InputStream = openStream(filePath) try { val arrayNode = mapper.readTree(is) if (!arrayNode.isArray) { @@ -578,7 +644,7 @@ package object file { throw ex } - val is: InputStream = HadoopInputFile.fromPath(new Path(filePath), conf).newStream() + val is: InputStream = openStream(filePath) try { val arrayNode = mapper.readTree(is) @@ -1106,9 +1172,14 @@ package object file { filePath: String )(implicit conf: Configuration = hadoopConfiguration): Boolean = { Try { - val fs = FileSystem.get(conf) - val deltaLogPath = new Path(filePath, "_delta_log") - fs.exists(deltaLogPath) && fs.getFileStatus(deltaLogPath).isDirectory + LocalPath.resolve(filePath) match { + case Some(local) => + Files.isDirectory(local.resolve("_delta_log")) + case None => + val fs = FileSystem.get(conf) + val deltaLogPath = new Path(filePath, "_delta_log") + fs.exists(deltaLogPath) && fs.getFileStatus(deltaLogPath).isDirectory + } }.getOrElse(false) } @@ -1118,7 +1189,7 @@ package object file { filePath: String )(implicit conf: Configuration = hadoopConfiguration): FileFormat = { Try { - val is = HadoopInputFile.fromPath(new Path(filePath), conf).newStream() + val is = openStream(filePath) try { val reader = new BufferedReader(new InputStreamReader(is, "UTF-8")) val firstChar = reader.read().toChar @@ -1173,13 +1244,15 @@ package object file { /** Returns a [[Configuration]] appropriate for the URI scheme embedded in `path`. */ def forPath(path: String): Configuration = { - val scheme = Try(new java.net.URI(path).getScheme).getOrElse(null) - val conf = scheme match { - case "s3a" | "s3" => s3aConf() - case "abfs" | "abfss" | "wasb" | "wasbs" => azureConf() - case "gs" => gcsConf() - case "hdfs" => hdfsConf() - case _ => localConf() + // Scheme detection is LocalPath's (AD-10): it lowercases, and — unlike `new URI` — it does + // not throw on an unencoded space, so `S3A://b/x` and `s3a://b/my file.jsonl` both reach the + // S3 branch instead of silently falling through to localConf() with no credentials. + val conf = LocalPath.scheme(path) match { + case Some("s3a") | Some("s3") => s3aConf() + case Some("abfs") | Some("abfss") | Some("wasb") | Some("wasbs") => azureConf() + case Some("gs") => gcsConf() + case Some("hdfs") => hdfsConf() + case _ => localConf() } loadUserXmlConf(conf) conf diff --git a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala index c303d39d..c06ef544 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala @@ -73,6 +73,136 @@ class FileSourceSpec extends AnyWordSpec with Matchers with ScalaFutures with Be result.last should include("Bob") } + "read a JSON lines file from a file:// URI" in { + val tempFile = java.io.File.createTempFile("test-uri", ".jsonl") + tempFile.deleteOnExit() + val writer = new java.io.PrintWriter(tempFile) + writer.println("""{"id":1,"name":"Alice"}""") + writer.println("""{"id":2,"name":"Bob"}""") + writer.close() + + val result = FileSourceFactory + .fromFile(tempFile.toPath.toUri.toString) + .runWith(Sink.seq) + .futureValue + + result should have size 2 + result.head should include("Alice") + } + + "read a JSON lines file whose path contains a space" in { + val dir = Files.createTempDirectory("test dir with space") + dir.toFile.deleteOnExit() + val tempFile = new File(dir.toFile, "customers.jsonl") + tempFile.deleteOnExit() + val writer = new java.io.PrintWriter(tempFile) + writer.println("""{"id":1,"name":"Alice"}""") + writer.close() + + // Schemeless + FileSourceFactory + .fromFile(tempFile.getAbsolutePath) + .runWith(Sink.seq) + .futureValue should have size 1 + + // file:// URI — java.io.File.toURI percent-encodes the space + FileSourceFactory + .fromFile(tempFile.toURI.toString) + .runWith(Sink.seq) + .futureValue should have size 1 + } + + "read a JSON lines file from a percent-encoded file:// URI" in { + // RED before the fix. Hadoop's Path re-encodes the '%' (toUri -> file:/…/test%2520dir/…) and + // then looks for a directory literally named "test%20dir", so this shape has ALWAYS failed + // with "File does not exist" — on every JDK. It is the shape java.io.File.toURI produces. + val dir = Files.createTempDirectory("test dir enc") + val tempFile = new File(dir.toFile, "customers.jsonl") + // Register the PARENT first. java.io.DeleteOnExitHook deletes in REVERSE registration order + // ("last in, first deleted"), and File.delete() silently fails on a non-empty directory — so + // the child must be registered LAST to be deleted FIRST, or the temp dir leaks every run. + dir.toFile.deleteOnExit() + tempFile.deleteOnExit() + val writer = new java.io.PrintWriter(tempFile) + writer.println("""{"id":1,"name":"Alice"}""") + writer.close() + + val encoded = tempFile.toURI.toString // contains %20 + encoded should include("%20") + + FileSourceFactory + .fromFile(encoded) + .runWith(Sink.seq) + .futureValue should have size 1 + } + + "read a JSON array file in memory from a file:// URI" in { + // Only coverage anywhere for JsonArrayFileSource.fromFileInMemory (site 4). + val tempFile = java.io.File.createTempFile("test-inmem", ".json") + tempFile.deleteOnExit() + val writer = new java.io.PrintWriter(tempFile) + writer.println("""[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]""") + writer.close() + + JsonArrayFileSource + .fromFileInMemory(tempFile.toPath.toUri.toString) + .runWith(Sink.seq) + .futureValue should have size 2 + } + + "read a JSON array file from a file:// URI" in { + val tempFile = java.io.File.createTempFile("test-uri-array", ".json") + tempFile.deleteOnExit() + val writer = new java.io.PrintWriter(tempFile) + writer.println("""[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]""") + writer.close() + + val result = FileSourceFactory + .fromFile(tempFile.toPath.toUri.toString, format = JsonArray) + .runWith(Sink.seq) + .futureValue + + result should have size 2 + result.last should include("Bob") + } + + "read a JSON lines file with Hadoop's local filesystem disabled" in { + // Issue #183 regression guard at the stream level: an unresolvable fs.file.impl means any + // Hadoop use for the `file` scheme throws. Passing means the fast path was taken. + // NOT `implicit` — it is passed explicitly below; making it implicit would collide with the + // class-level `implicit val conf`. + val poisoned: Configuration = { + val c = new Configuration() + c.set("fs.file.impl", "does.not.Exist") + // Mandatory: without this, Hadoop's static FileSystem cache (keyed on scheme/authority/ugi, + // NOT on the Configuration) returns the LocalFileSystem that this suite's Parquet and Delta + // tests already created in this same JVM, `fs.file.impl` is never read, and this test + // passes even with the bug present. See LocalPathSpec for the bytecode evidence. + c.setBoolean("fs.file.impl.disable.cache", true) + c + } + + // Guard self-check: prove the poisoning is armed in THIS JVM before trusting the assertion. + val probe = java.io.File.createTempFile("test-nohadoop-probe", ".jsonl") + probe.deleteOnExit() + intercept[Exception] { + org.apache.parquet.hadoop.util.HadoopInputFile + .fromPath(new Path(probe.getAbsolutePath), poisoned) + .newStream() + }.toString should include("does.not.Exist") + + val tempFile = java.io.File.createTempFile("test-nohadoop", ".jsonl") + tempFile.deleteOnExit() + val writer = new java.io.PrintWriter(tempFile) + writer.println("""{"id":1,"name":"Alice"}""") + writer.close() + + FileSourceFactory + .fromFile(tempFile.getAbsolutePath)(ec, poisoned) + .runWith(Sink.seq) + .futureValue should have size 1 + } + "read JSON Array file (single line)" in { val tempFile = java.io.File.createTempFile("test", ".json") tempFile.deleteOnExit() diff --git a/core/src/test/scala/app/softnetwork/elastic/client/file/LocalPathSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/file/LocalPathSpec.scala new file mode 100644 index 00000000..51fade8c --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/file/LocalPathSpec.scala @@ -0,0 +1,412 @@ +package app.softnetwork.elastic.client.file + +import app.softnetwork.elastic.sql.query.{Delta, Json, JsonArray, Unknown} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.scalatest.OptionValues +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths} + +// `OptionValues` is what supplies `.value` on an Option — without it this file does not compile. +class LocalPathSpec extends AnyWordSpec with Matchers with OptionValues { + + private def str(p: java.nio.file.Path): String = p.toString.replace('\\', '/') + + "LocalPath.resolve" should { + + "reject blank input" in { + LocalPath.resolve(null) shouldBe None + LocalPath.resolve("") shouldBe None + LocalPath.resolve(" ") shouldBe None + } + + "accept a schemeless absolute path" in { + str(LocalPath.resolve("/Users/me/data/customers.jsonl").value) shouldBe + "/Users/me/data/customers.jsonl" + } + + "accept schemeless relative paths" in { + str(LocalPath.resolve("data/customers.jsonl").value) shouldBe "data/customers.jsonl" + str(LocalPath.resolve("./data/customers.jsonl").value) shouldBe "./data/customers.jsonl" + } + + "accept a schemeless path containing spaces" in { + str(LocalPath.resolve("/Users/me/my data/customers.jsonl").value) shouldBe + "/Users/me/my data/customers.jsonl" + } + + "accept the three-slash file URI form" in { + str(LocalPath.resolve("file:///Users/me/data/customers.jsonl").value) shouldBe + "/Users/me/data/customers.jsonl" + } + + "accept the single-slash file URI form" in { + str(LocalPath.resolve("file:/Users/me/data/customers.jsonl").value) shouldBe + "/Users/me/data/customers.jsonl" + } + + "accept a localhost authority" in { + str(LocalPath.resolve("file://localhost/Users/me/data/customers.jsonl").value) shouldBe + "/Users/me/data/customers.jsonl" + } + + "defer a non-local authority to Hadoop" in { + // `file://Users/...` parses `Users` as the authority — NOT this machine. + LocalPath.resolve("file://Users/me/data/customers.jsonl") shouldBe None + } + + "accept a file URI containing an unencoded space" in { + // `new URI` throws here; the literal fallback must keep the space verbatim. + str(LocalPath.resolve("file:///Users/me/my data/customers.jsonl").value) shouldBe + "/Users/me/my data/customers.jsonl" + } + + "percent-decode a file URI" in { + str(LocalPath.resolve("file:///Users/me/my%20data/customers.jsonl").value) shouldBe + "/Users/me/my data/customers.jsonl" + } + + "percent-decode non-ASCII characters as UTF-8" in { + // Unicode ESCAPES, not literal accents: scalacCompilerOptions (build.sbt:13-17) is + // Seq("-deprecation", "-feature", "-target:jvm-1.8") — there is NO `-encoding UTF-8`, so + // source decoding follows the platform default charset. A literal "été" here would make the + // assertion machine-dependent. + str(LocalPath.resolve("file:///Users/me/data/%C3%A9t%C3%A9.jsonl").value) shouldBe + "/Users/me/data/\u00e9t\u00e9.jsonl" + } + + "not turn '+' into a space (URLDecoder trap)" in { + str(LocalPath.resolve("file:///Users/me/data/my+file.jsonl").value) shouldBe + "/Users/me/data/my+file.jsonl" + } + + "treat a Windows drive letter as a path, not a scheme" in { + str(LocalPath.resolve("""C:\data\customers.jsonl""").value) should startWith("C:") + str(LocalPath.resolve("C:/data/customers.jsonl").value) should startWith("C:") + } + + "resolve a Windows file URI according to the host platform" in { + // The drive-slash strip is platform-gated (see stripDriveSlash), so assert what THIS OS does. + // The platform-independent assertion lives in the "strip a Windows drive slash" test below. + val resolved = str(LocalPath.resolve("file:///C:/data/customers.jsonl").value) + if (LocalPath.onWindows) resolved should startWith("C:") + else resolved shouldBe "/C:/data/customers.jsonl" + } + + "not expand a tilde" in { + str(LocalPath.resolve("~/data/customers.jsonl").value) shouldBe "~/data/customers.jsonl" + } + + "accept an uppercase FILE scheme" in { + str(LocalPath.resolve("FILE:///Users/me/data/customers.jsonl").value) shouldBe + "/Users/me/data/customers.jsonl" + } + + "reject a file URI with no path" in { + LocalPath.resolve("file:") shouldBe None + LocalPath.resolve("file://") shouldBe None + } + + "handle an opaque file URI via the literal fallback" in { + // `new URI` SUCCEEDS here but getPath is null — the guard must route to splitLiteral. + str(LocalPath.resolve("file:relative/x.jsonl").value) shouldBe "relative/x.jsonl" + } + + "keep '?' and '#' as file name characters, exactly like Hadoop" in { + // Verified against hadoop-common 3.4.2: new Path("file:///a/report#1.jsonl").toUri.getPath + // is "/a/report#1.jsonl". Taking URI.getPath here would read "/a/report" — the WRONG FILE. + str(LocalPath.resolve("file:///a/report#1.jsonl").value) shouldBe "/a/report#1.jsonl" + str(LocalPath.resolve("file:///a/b.jsonl?x=1").value) shouldBe "/a/b.jsonl?x=1" + str(LocalPath.resolve("/Users/me/data/report#1.jsonl").value) shouldBe + "/Users/me/data/report#1.jsonl" + } + + "NOT trim surrounding whitespace" in { + // Hadoop preserves it (new Path("/tmp/trailing ").toUri.getPath == "/tmp/trailing "). + // Trimming would silently address a different file. + str(LocalPath.resolve("/tmp/trailing ").value) shouldBe "/tmp/trailing " + str(LocalPath.resolve(" /Users/me/x.jsonl").value) shouldBe " /Users/me/x.jsonl" + } + + "defer every remote scheme to Hadoop" in { + LocalPath.resolve("s3a://bucket/customers.jsonl") shouldBe None + LocalPath.resolve("s3://bucket/customers.jsonl") shouldBe None + LocalPath.resolve("gs://bucket/customers.jsonl") shouldBe None + LocalPath.resolve("hdfs://namenode:8020/customers.jsonl") shouldBe None + LocalPath.resolve("abfss://c@a.dfs.core.windows.net/customers.jsonl") shouldBe None + LocalPath.resolve("wasbs://c@a.blob.core.windows.net/customers.jsonl") shouldBe None + } + } + + // --------------------------------------------------------------------------------------------- + // Issue #183 regression guard. + // + // A Configuration whose `fs.file.impl` points at a class that does not exist makes any Hadoop + // FileSystem resolution for the `file` scheme blow up with + // RuntimeException: java.lang.ClassNotFoundException: Class does.not.Exist not found + // So: if these tests pass, the local fast path was taken and Hadoop was never consulted — proven + // on every JDK, including the JDK 11 that CI runs today. This is what stops the bug coming back. + // + // BOTH properties are load-bearing — `fs.file.impl` alone is NOT enough. + // `FileSystem$Cache$Key` is (scheme, authority, ugi, unique); the Configuration is NOT part of + // the key (verified: `javap -c` on hadoop-common 3.4.2, `FileSystem$Cache$Key.`). Once ANY + // code in this JVM has resolved a `file`-scheme FileSystem with a working Configuration, the + // cache hands the same LocalFileSystem back and `fs.file.impl` is never read again — the guard + // silently becomes vacuous. `core` tests run UNFORKED in one JVM by default, and + // `FileSourceSpec`'s Parquet + Delta tests populate that cache. + // `fs.file.impl.disable.cache = true` makes `FileSystem.get(uri, conf)` call `createFileSystem` + // BEFORE consulting the cache (verified in the same disassembly: the `fs.%s.impl.disable.cache` + // branch precedes `CACHE.get`), so the poisoned class name is always resolved and always throws. + // --------------------------------------------------------------------------------------------- + "the local fast path" should { + + def poisoned: Configuration = { + val conf = new Configuration() + conf.set("fs.file.impl", "does.not.Exist") + conf.setBoolean("fs.file.impl.disable.cache", true) // see the note above — do not remove + conf + } + + def writeTemp(suffix: String, content: String): File = { + val f = File.createTempFile("localpath", suffix) + f.deleteOnExit() + Files.write(f.toPath, content.getBytes(StandardCharsets.UTF_8)) + f + } + + // Meta-test: proves the guard itself is ARMED. Without it, every assertion below could pass + // while Hadoop was happily consulted, and nobody would notice. This test must be RED if the + // poisoning ever stops working (wrong property name, Hadoop behaviour change, cache hit). + "actually be able to detect a Hadoop call (guard self-check)" in { + val f = writeTemp(".jsonl", """{"id":0}""" + "\n") + val ex = intercept[Exception] { + HadoopInputFile.fromPath(new Path(f.getAbsolutePath), poisoned).newStream() + } + // RuntimeException: java.lang.ClassNotFoundException: Class does.not.Exist not found + ex.toString should include("does.not.Exist") + } + + "open a local stream without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val f = writeTemp(".jsonl", """{"id":1}""" + "\n") + + val is = openStream(f.getAbsolutePath) + try new String(is.readAllBytes(), StandardCharsets.UTF_8) should include("\"id\":1") + finally is.close() + } + + "open a file:// URI without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val f = writeTemp(".jsonl", """{"id":2}""" + "\n") + + val is = openStream(f.toPath.toUri.toString) + try new String(is.readAllBytes(), StandardCharsets.UTF_8) should include("\"id\":2") + finally is.close() + } + + "sniff a JSON array without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val f = writeTemp(".json", """[{"id":1},{"id":2}]""") + + FileFormatDetector.detect(f.getAbsolutePath) shouldBe JsonArray + } + + "sniff JSON lines without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val f = writeTemp(".json", """{"id":1}""" + "\n") + + FileFormatDetector.detect(f.getAbsolutePath) shouldBe Json + } + + "classify an unknown local extension without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val f = writeTemp(".txt", "not json") + + // Exercises isDeltaTable (site 7): no _delta_log ⇒ Unknown, and NO Hadoop call. + FileFormatDetector.detect(f.getAbsolutePath) shouldBe Unknown + } + + "detect a local Delta table directory without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val dir = Files.createTempDirectory("localpath-delta") + val log = Files.createDirectory(dir.resolve("_delta_log")) + // Register the PARENT first. java.io.DeleteOnExitHook deletes in REVERSE registration order + // ("last in, first deleted"), and File.delete() silently fails on a non-empty directory — so + // the child must be registered LAST to be deleted FIRST, or the temp dir leaks every run. + dir.toFile.deleteOnExit() + log.toFile.deleteOnExit() + + FileFormatDetector.detect(dir.toAbsolutePath.toString) shouldBe Delta + } + + "report a local directory with the historical message" in { + implicit val conf: Configuration = poisoned + val dir = Files.createTempDirectory("localpath-dir") + dir.toFile.deleteOnExit() + + // Covers validateLocalPath's `Path is not a file` branch, which nothing else exercises. + val ex = intercept[IllegalArgumentException] { + JsonArrayFileSource.getMetadata(dir.toAbsolutePath.toString) + } + ex.getMessage should include("Path is not a file") + } + + "route a file:// URI with a non-local authority through Hadoop" in { + // Covers validateHadoopPath, which is otherwise DEAD in the test suite after this change: + // every other path in core's tests is local. Uses a clean Configuration on purpose. + // + // MEASURED on both JDKs, not assumed. The invariant under test is "this input still reaches + // Hadoop"; WHICH way Hadoop then fails is JDK-dependent, and both ways prove the routing: + // + // JDK <= 22 : IllegalArgumentException + // "Wrong FS: file://someotherhost/no-such-183.json, expected: file:///" + // — LocalFileSystem.checkPath rejects the foreign authority before stat'ing. + // (NOT "does not exist": checkPath runs first.) + // JDK >= 23 : UnsupportedOperationException "getSubject is not supported" + // — FileSystem.get -> CACHE.get -> new Key -> UGI.getCurrentUser. This suite + // is run forked onto JDK 25 by AC 1b, so that branch is really exercised. + // + // The local fast path can produce NEITHER string, so either one proves validateHadoopPath + // ran. Asserting only the JDK-11 message would make this test fail on the very JDK the + // story exists to support. + implicit val conf: Configuration = hadoopConfiguration + val ex = intercept[Exception] { + JsonArrayFileSource.getMetadata("file://someotherhost/no-such-183.json") + } + withClue(s"unexpected failure mode: $ex") { + ex.toString should (include("Wrong FS") or include("getSubject")) + } + } + + "validate a local JSON array file without consulting Hadoop" in { + implicit val conf: Configuration = poisoned + val f = writeTemp(".json", """[{"id":1},{"id":2},{"id":3}]""") + + // getMetadata calls validateFile (site 1) then opens the stream (site 5). + JsonArrayFileSource.getMetadata(f.getAbsolutePath).elementCount shouldBe 3 + } + + "report a missing local file with the historical message" in { + implicit val conf: Configuration = poisoned + val missing = Paths.get(System.getProperty("java.io.tmpdir"), "no-such-183.json").toString + + val ex = intercept[IllegalArgumentException] { + JsonArrayFileSource.getMetadata(missing) + } + ex.getMessage should include("does not exist") + ex.getMessage should include(missing) + } + + "still route a remote scheme through Hadoop" in { + // Do NOT call openStream("s3a://…") here. `hadoop-aws` is `% Provided` + // (core/build.sbt:46) and sbt puts Provided on the TEST classpath, so S3AFileSystem really + // instantiates, runs the default AWS credential chain (including an IMDS probe at + // 169.254.169.254) and issues a live request for a bucket named `bucket`. That is a + // network-dependent unit test that hangs offline and can throw an Error (not an Exception) + // on SDK skew. Classification is what this test is about, and it is pure: + LocalPath.resolve("s3a://bucket/customers.jsonl") shouldBe None + } + } + + // --------------------------------------------------------------------------------------------- + // AD-10 — `HadoopConfigurationFactory.forPath` is now rewired onto `LocalPath.scheme`, so there + // is ONE scheme parser in this package instead of two that disagreed. Pure `Configuration` + // assertions only: never open a stream or call `FileSystem.get` for a remote scheme here (see + // the note on "still route a remote scheme through Hadoop" above). + // --------------------------------------------------------------------------------------------- + "HadoopConfigurationFactory.forPath" should { + "route an UPPERCASE remote scheme to its cloud configuration (was: localConf, no credentials)" in { + HadoopConfigurationFactory.forPath("S3A://bucket/x.jsonl").get("fs.s3a.impl") shouldBe + "org.apache.hadoop.fs.s3a.S3AFileSystem" + } + + "route a remote scheme whose path contains an unencoded space (was: localConf)" in { + // `new java.net.URI(...)` throws on this input, which is exactly how the credentials were lost. + HadoopConfigurationFactory.forPath("s3a://bucket/my file.jsonl").get("fs.s3a.impl") shouldBe + "org.apache.hadoop.fs.s3a.S3AFileSystem" + } + + "still treat file: and schemeless paths as local" in { + HadoopConfigurationFactory.forPath("/tmp/x.jsonl").get("fs.file.impl") shouldBe + "org.apache.hadoop.fs.LocalFileSystem" + HadoopConfigurationFactory.forPath("file:///tmp/x.jsonl").get("fs.file.impl") shouldBe + "org.apache.hadoop.fs.LocalFileSystem" + } + + "still treat a Windows drive letter as local, not as scheme 'C'" in { + HadoopConfigurationFactory.forPath("C:/data/x.jsonl").get("fs.file.impl") shouldBe + "org.apache.hadoop.fs.LocalFileSystem" + } + } + + "LocalPath.scheme" should { + "lowercase, and refuse a one-character prefix" in { + LocalPath.scheme("S3A://b/x") shouldBe Some("s3a") + LocalPath.scheme("FILE:///tmp/x") shouldBe Some("file") + LocalPath.scheme("s3a://b/my file.jsonl") shouldBe Some("s3a") // `new URI` would throw here + LocalPath.scheme("C:/data/x.jsonl") shouldBe None + LocalPath.scheme("/tmp/x.jsonl") shouldBe None + LocalPath.scheme(null) shouldBe None + } + } + + // --------------------------------------------------------------------------------------------- + // Differential parity — the ONLY assertion that pins "bit-identical to Hadoop on JDK ≤ 22", + // which AD-6, AD-8 and the regression-risk table all claim. Runs on the default JDK 11, where + // Hadoop still works, so it is a real comparison and not a restatement of the new code. + // Deliberately NOT run under -Dtest.jdk.home=<24|25>: there Hadoop cannot answer at all. + // --------------------------------------------------------------------------------------------- + "LocalPath.resolve" should { + "agree with org.apache.hadoop.fs.Path on every local form" in { + val cases = Seq( + "/Users/me/data/customers.jsonl", + "/Users/me/my data/customers.jsonl", + "/Users/me/data/report#1.jsonl", + "/Users/me/data/a?b.jsonl", + "/tmp/trailing ", + "data/customers.jsonl", + "./data/customers.jsonl", + "~/data/customers.jsonl", + "file:///Users/me/data/customers.jsonl", + "file:/Users/me/data/customers.jsonl", + "file://localhost/Users/me/data/customers.jsonl", + "file:///Users/me/my data/customers.jsonl", + "file:///a/report#1.jsonl", + "file:///a/b.jsonl?x=1", + "file:///Users/me/data/my+file.jsonl" + ) + cases.foreach { s => + withClue(s"input [$s]: ") { + // `.normalize()` on BOTH sides: Hadoop's Path collapses a leading "./" while + // java.nio.file.Paths keeps it. `./data/x.jsonl` and `data/x.jsonl` denote the same file + // against the same working directory, so that difference is cosmetic — this comparison + // asserts "same file", which is the property AD-6/AD-8 actually claim. + // Measured: 15 inputs, 14 byte-identical, 1 ("./data/customers.jsonl") equal only after + // normalize. If a SECOND input ever needs normalize to agree, investigate — do not widen. + val mine = LocalPath.resolve(s).map(p => Paths.get(p.toString).normalize.toString) + val hadoop = Paths.get(new Path(s).toUri.getPath).normalize.toString + mine shouldBe Some(hadoop) + } + } + } + + "diverge from Hadoop ONLY on percent-decoding, and only in the user's favour" in { + // java.io.File.toURI emits this shape; Hadoop re-encodes the '%' and looks for a directory + // literally named "my%20data", which is why that form has ALWAYS failed. We decode it. + val encoded = "file:///Users/me/my%20data/customers.jsonl" + new Path(encoded).toUri.getPath shouldBe "/Users/me/my%20data/customers.jsonl" + str(LocalPath.resolve(encoded).value) shouldBe "/Users/me/my data/customers.jsonl" + } + + "strip a Windows drive slash only on Windows" in { + LocalPath.stripDriveSlash("/C:/data/x.jsonl", windows = true) shouldBe "C:/data/x.jsonl" + LocalPath.stripDriveSlash("/C:/data/x.jsonl", windows = false) shouldBe "/C:/data/x.jsonl" + LocalPath.stripDriveSlash("/Users/me/x.jsonl", windows = true) shouldBe "/Users/me/x.jsonl" + } + } +} diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md index 2c2859ea..989a2407 100644 --- a/documentation/sql/dml_statements.md +++ b/documentation/sql/dml_statements.md @@ -391,6 +391,59 @@ This allows fine-grained property overrides without changing environment variabl ``` +#### JVM compatibility — JDK 23 and newer + +`JSON`, `JSON_ARRAY` and automatic format detection read **local** files +(`/path/to/file.jsonl`, `file:///path/to/file.jsonl`) directly through `java.nio.file`, so they work +on every supported JVM, **including JDK 23, 24 and 25**. + +`PARQUET`, `DELTA_LAKE` and **every remote scheme** go through Apache Hadoop, whose +`UserGroupInformation.getCurrentUser()` calls +`javax.security.auth.Subject.getSubject(AccessControlContext)`. That method was re-specified in +**JDK 23** to throw whenever a Security Manager is not allowed (which is the default), and +[JEP 486](https://openjdk.org/jeps/486) made it throw *unconditionally* in **JDK 24** while removing +the escape hatch — a JDK 24+ launcher refuses to start if `java.security.manager` is set. Those +paths therefore fail with: + +| JDK | Error text | `-Djava.security.manager=allow` | +|---|---|---| +| 23 | `getSubject is supported only if a security manager is allowed` | works around it | +| 24 and newer | `getSubject is not supported` | JVM refuses to start | + +| `COPY INTO` source | JDK 8 – 22 | JDK 23 | JDK 24+ | +|---|---|---|---| +| local `JSON` / `JSON_ARRAY`, schemeless or `file:` | ✔ | ✔ | ✔ | +| local file with auto-detected format | ✔ | ✔ | ✔ | +| local `PARQUET` | ✔ | ✖ * | ✖ | +| local `DELTA_LAKE` | ✔ | ✖ * | ✖ | +| any remote scheme (`s3a://`, `s3://`, `gs://`, `abfs*://`, `wasb*://`, `hdfs://`) | ✔ | ✖ * | ✖ | + +\* works on JDK 23 if the host process is started with `-Djava.security.manager=allow`. + +**Workaround for the unsupported combinations:** run the host process on **JDK 21** (or any +JDK ≤ 22). On JDK 23 you may instead add `-Djava.security.manager=allow`; on JDK 24+ there is no +flag that helps. For DBeaver, add this to `dbeaver.ini` **before** `-vmargs` (a DBeaver update +overwrites the file): + +``` +-vm +/path/to/jdk-21/Contents/Home/lib/libjli.dylib +``` + +Lifting the Parquet / Delta / remote restriction depends on an upstream Hadoop release that no +longer calls `Subject.getSubject`. Tracked as +[SoftClient4ES#183](https://github.com/SOFTNETWORK-APP/SoftClient4ES/issues/183). + +**Local path notes:** `~` is **not** expanded — pass an absolute path. Relative paths resolve +against the working directory of the process running the query. A `file://` URI may percent-encode +special characters (`file:///data/my%20file.jsonl`); an unencoded space is accepted too. Leading and +trailing whitespace around the path is ignored. Wildcards/globs (`/data/*.jsonl`) and transparent +`.gz` decompression are **not** supported — they never were, on any JDK. + +Note that percent-encoded `file:` URIs are decoded only on the local `JSON` / `JSON_ARRAY` fast +path. `PARQUET` and `DELTA_LAKE` still go through Hadoop, which takes `%20` literally — pass a +schemeless path for those formats. + --- ## DML Lifecycle Example diff --git a/documentation/sql/known_limitations.md b/documentation/sql/known_limitations.md index 9dbb6a3d..e022f6ea 100644 --- a/documentation/sql/known_limitations.md +++ b/documentation/sql/known_limitations.md @@ -51,6 +51,16 @@ WHERE department_id IN (SELECT id FROM departments WHERE region = 'EU'); The parser rejects this — `IN` accepts only literal value lists today, not a nested `SELECT`. Rewrite it as an explicit JOIN (fully supported), or wait for the next release where the subquery form lands as-is. +## Runtime / JVM limitations + +These are constraints of the host JVM, not unimplemented SQL features — they have no delivery date because they depend on upstream projects. + +- **`COPY INTO` with `PARQUET`, `DELTA_LAKE`, or a remote URI (`s3a://`, `gs://`, `abfs://`, + `hdfs://`) does not work on JDK 23 or newer.** Local `JSON` / `JSON_ARRAY` files do, on every JDK. + This is an Apache Hadoop limitation (JDK 23 re-specified, and JEP 486 in JDK 24 removed, the API + Hadoop's `UserGroupInformation` depends on); run the host process on JDK 21 for those sources. See + [DML statements → COPY INTO → JVM compatibility](dml_statements.md#jvm-compatibility--jdk-23-and-newer). + ## Coming in the upcoming release (Quarter 1 2027) - **Heterogeneous federation**: JOIN or correlate Elasticsearch with PostgreSQL, MySQL, ClickHouse, Snowflake, and more — plus cross-cluster subqueries (e.g. correlate one cluster's data against another's). diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala index 7fa06f88..ea5d8f22 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala @@ -516,6 +516,58 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { ) } + it should "support COPY INTO from a file:// URI and from a path containing a space" in { + val create = + """CREATE TABLE IF NOT EXISTS copy_into_uri_test ( + | uuid KEYWORD NOT NULL, + | name VARCHAR, + | PRIMARY KEY (uuid) + |)""".stripMargin + + assertDdl(System.nanoTime(), executeSync(create)) + + // 1. file:/// URI form — the exact shape reported in SoftClient4ES#183 + val uriFile = java.io.File.createTempFile("copy_into_uri", ".jsonl") + uriFile.deleteOnExit() + val w1 = new java.io.PrintWriter(uriFile) + w1.println("""{"uuid": "U1", "name": "Homer Simpson"}""") + w1.println("""{"uuid": "U2", "name": "Moe Szyslak"}""") + w1.close() + + val copyUri = s"""COPY INTO copy_into_uri_test FROM "${uriFile.toPath.toUri.toString}"""" + assertDml(System.nanoTime(), executeSync(copyUri), Some(DmlResult(inserted = 2))) + + // 2. a `file:` URI whose path contains an UNENCODED space. `new java.net.URI(...)` throws on + // this input, so it is the only statement here that exercises LocalPath's literal fallback, + // and it is RED before the fix (Hadoop's Path handles it, but only via the schemeless form). + // A schemeless spaced path would NOT prove this — schemeless never reaches `new URI` at all. + val spacedDir = java.nio.file.Files.createTempDirectory("copy into dir") + spacedDir.toFile.deleteOnExit() + val spacedFile = new java.io.File(spacedDir.toFile, "customers.jsonl") + spacedFile.deleteOnExit() + val w2 = new java.io.PrintWriter(spacedFile) + w2.println("""{"uuid": "U3", "name": "Barney Gumble"}""") + w2.close() + + // NOTE: `file://` + the RAW absolute path, i.e. the space is left unencoded on purpose. + // Do not substitute `spacedFile.toURI.toString` here — that percent-encodes it and exercises a + // different branch (already covered by FileSourceSpec). + val copySpaced = + s"""COPY INTO copy_into_uri_test FROM "file://${spacedFile.getAbsolutePath}" ON CONFLICT DO UPDATE""" + assertDml(System.nanoTime(), executeSync(copySpaced), Some(DmlResult(inserted = 1))) + + val select = executeSync("SELECT * FROM copy_into_uri_test ORDER BY uuid ASC") + assertSelectResult( + System.nanoTime(), + select, + Seq( + Map("uuid" -> "U1", "name" -> "Homer Simpson"), + Map("uuid" -> "U2", "name" -> "Moe Szyslak"), + Map("uuid" -> "U3", "name" -> "Barney Gumble") + ) + ) + } + // ========================================================================= // 5. DQL — SELECT, JOIN, UNNEST, GROUP BY, etc. // =========================================================================